You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/03/13 15:12:18 UTC
(doris) branch branch-4.0 updated: branch-4.0: [feature](join) support ASOF join (#59591) (#61320)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 7fdcf736a16 branch-4.0: [feature](join) support ASOF join (#59591) (#61320)
7fdcf736a16 is described below
commit 7fdcf736a166c390d3b046b5536a5c51db88da31
Author: zclllyybb <zh...@selectdb.com>
AuthorDate: Fri Mar 13 23:12:08 2026 +0800
branch-4.0: [feature](join) support ASOF join (#59591) (#61320)
pick https://github.com/apache/doris/pull/59591
Co-authored-by: starocean999 <li...@selectdb.com>
---
be/src/common/status.h | 8 +
be/src/pipeline/common/join_utils.h | 132 +-
be/src/pipeline/dependency.h | 12 +
be/src/pipeline/exec/hashjoin_build_sink.cpp | 223 ++-
be/src/pipeline/exec/hashjoin_build_sink.h | 14 +
be/src/pipeline/exec/hashjoin_probe_operator.cpp | 42 +-
be/src/pipeline/exec/hashjoin_probe_operator.h | 15 +-
.../exec/join/asof_left_inner_join_impl.cpp | 26 +
.../exec/join/asof_left_outer_join_impl.cpp | 26 +
.../pipeline/exec/join/process_hash_table_probe.h | 11 +
.../exec/join/process_hash_table_probe_impl.h | 280 +++-
be/src/pipeline/exec/join_build_sink_operator.cpp | 7 +-
be/src/pipeline/exec/join_probe_operator.cpp | 3 +-
be/src/vec/common/hash_table/join_hash_table.h | 17 +-
be/test/pipeline/operator/asof_join_test.cpp | 387 +++++
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 2 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 8 +-
.../org/apache/doris/analysis/JoinOperator.java | 68 +-
.../glue/translator/PhysicalPlanTranslator.java | 56 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 64 +-
.../processor/post/RuntimeFilterGenerator.java | 1 +
.../post/RuntimeFilterPushDownVisitor.java | 3 +
.../processor/post/TopnFilterPushDownVisitor.java | 2 +-
.../post/materialize/LazySlotPruning.java | 6 +-
.../post/runtimefilterv2/PushDownVisitor.java | 2 +-
.../properties/ChildOutputPropertyDeriver.java | 8 +
.../properties/ChildrenPropertiesRegulator.java | 1 +
.../nereids/rules/analysis/BindExpression.java | 56 +-
.../rules/exploration/join/JoinCommute.java | 5 +-
.../join/PushDownProjectThroughInnerOuterJoin.java | 16 +-
.../mv/AbstractMaterializedViewRule.java | 6 +-
.../rules/exploration/mv/HyperGraphComparator.java | 3 +
.../nereids/rules/rewrite/ConstantPropagation.java | 3 +
.../rules/rewrite/ConvertOuterJoinToAntiJoin.java | 3 +-
.../rules/rewrite/EliminateEmptyRelation.java | 8 +-
.../nereids/rules/rewrite/EliminateOuterJoin.java | 10 +-
.../nereids/rules/rewrite/InferJoinNotNull.java | 5 +-
.../nereids/rules/rewrite/InferPredicates.java | 11 +-
.../doris/nereids/rules/rewrite/InitJoinOrder.java | 10 +-
.../nereids/rules/rewrite/PullUpPredicates.java | 11 +-
.../rules/rewrite/PullUpProjectUnderLimit.java | 3 +-
.../rules/rewrite/PullUpProjectUnderTopN.java | 5 +-
.../rules/rewrite/PushDownFilterThroughJoin.java | 8 +-
.../doris/nereids/rules/rewrite/PushDownLimit.java | 2 +
.../rewrite/PushDownLimitDistinctThroughJoin.java | 2 +
.../rewrite/PushDownTopNDistinctThroughJoin.java | 6 +-
.../rules/rewrite/PushDownTopNThroughJoin.java | 2 +
.../rules/rewrite/PushFilterInsideJoin.java | 3 +-
.../doris/nereids/rules/rewrite/ReorderJoin.java | 5 +-
.../doris/nereids/rules/rewrite/SaltJoin.java | 51 +-
.../doris/nereids/rules/rewrite/SkewJoin.java | 7 +-
.../rewrite/TransposeSemiJoinLogicalJoin.java | 5 +-
.../TransposeSemiJoinLogicalJoinProject.java | 5 +-
.../apache/doris/nereids/stats/JoinEstimation.java | 106 +-
.../apache/doris/nereids/trees/plans/JoinType.java | 53 +
.../nereids/trees/plans/logical/LogicalJoin.java | 48 +-
.../trees/plans/logical/LogicalUsingJoin.java | 27 +-
.../trees/plans/physical/PhysicalHashJoin.java | 8 +-
.../org/apache/doris/nereids/util/JoinUtils.java | 10 +-
.../org/apache/doris/planner/HashJoinNode.java | 16 +-
.../apache/doris/planner/NestedLoopJoinNode.java | 4 +-
.../doris/nereids/memo/StructInfoMapTest.java | 12 +-
.../doris/nereids/parser/NereidsParserTest.java | 10 +
.../properties/ChildOutputPropertyDeriverTest.java | 22 +-
.../nereids/rules/analysis/BindExpressionTest.java | 2 +-
.../rules/analysis/BindSlotReferenceTest.java | 5 +-
.../rules/exploration/join/JoinCommuteTest.java | 63 +-
.../PushDownProjectThroughInnerOuterJoinTest.java | 118 +-
.../rules/rewrite/ConstantPropagationTest.java | 28 +
.../rewrite/ConvertOuterJoinToAntiJoinTest.java | 86 +-
.../EliminateConstHashJoinConditionTest.java | 16 +
.../rules/rewrite/EliminateJoinByFkTest.java | 21 +
.../rules/rewrite/EliminateJoinByUniqueTest.java | 30 +-
.../rules/rewrite/EliminateOuterJoinTest.java | 27 +-
.../nereids/rules/rewrite/InferPredicatesTest.java | 56 +-
.../rewrite/PushDownFilterThroughJoinTest.java | 21 +-
.../rewrite/PushDownJoinOtherConditionTest.java | 108 +-
.../PushDownLimitDistinctThroughJoinTest.java | 42 +-
.../rules/rewrite/PushDownTopNThroughJoinTest.java | 28 +-
.../rules/rewrite/PushFilterInsideJoinTest.java | 2 +-
.../nereids/rules/rewrite/ReorderJoinTest.java | 59 +-
.../doris/nereids/rules/rewrite/SaltJoinTest.java | 77 +-
.../doris/nereids/stats/JoinEstimateTest.java | 42 +
gensrc/thrift/PlanNodes.thrift | 7 +-
.../data/nereids_p0/join/asof/test_asof_join.out | 327 ++++
.../join/asof/test_asof_join_build_expr_edge.out | 79 +
.../join/asof/test_asof_join_explicit.out | 97 ++
.../nereids_p0/join/asof/test_asof_join_expr.out | 107 ++
.../join/asof/test_asof_join_large_bucket.out | 35 +
.../nereids_p0/join/asof/test_asof_join.groovy | 1621 ++++++++++++++++++++
.../asof/test_asof_join_build_expr_edge.groovy | 401 +++++
.../join/asof/test_asof_join_explicit.groovy | 369 +++++
.../join/asof/test_asof_join_expr.groovy | 477 ++++++
.../join/asof/test_asof_join_large_bucket.groovy | 145 ++
94 files changed, 6144 insertions(+), 343 deletions(-)
diff --git a/be/src/common/status.h b/be/src/common/status.h
index fef94ca0ca8..1b943d71144 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -757,6 +757,14 @@ using ResultError = unexpected<Status>;
std::forward<T>(res).value(); \
})
+// core in Debug mode, exception in Release mode.
+#define DORIS_CHECK(stmt) \
+ do { \
+ if (!static_cast<bool>(stmt)) [[unlikely]] { \
+ throw Exception(Status::FatalError(fmt::format("Check failed: {}", #stmt))); \
+ } \
+ } while (false)
+
} // namespace doris
// specify formatter for Status
diff --git a/be/src/pipeline/common/join_utils.h b/be/src/pipeline/common/join_utils.h
index 8d46c317fb5..7ea6fe6c128 100644
--- a/be/src/pipeline/common/join_utils.h
+++ b/be/src/pipeline/common/join_utils.h
@@ -17,6 +17,7 @@
#pragma once
+#include <algorithm>
#include <variant>
#include "vec/common/hash_table/hash_key_type.h"
@@ -24,6 +25,27 @@
#include "vec/common/hash_table/join_hash_table.h"
namespace doris {
+
+// Devirtualize compare_at for ASOF JOIN supported column types.
+// ASOF JOIN only supports DateV2, DateTimeV2, and TimestampTZ.
+// Dispatches to the concrete ColumnVector<T> once so that all compare_at
+// calls inside `func` are direct (non-virtual) calls.
+// `func` receives a single argument: a const pointer to the concrete column
+// (or const IColumn* as fallback for unexpected types).
+template <typename Func>
+decltype(auto) asof_column_dispatch(const vectorized::IColumn* col, Func&& func) {
+ if (const auto* c_dv2 = vectorized::check_and_get_column<vectorized::ColumnDateV2>(col)) {
+ return std::forward<Func>(func)(c_dv2);
+ } else if (const auto* c_dtv2 =
+ vectorized::check_and_get_column<vectorized::ColumnDateTimeV2>(col)) {
+ return std::forward<Func>(func)(c_dtv2);
+ } else if (const auto* c_tstz =
+ vectorized::check_and_get_column<vectorized::ColumnTimeStampTz>(col)) {
+ return std::forward<Func>(func)(c_tstz);
+ } else {
+ return std::forward<Func>(func)(col);
+ }
+}
using JoinOpVariants =
std::variant<std::integral_constant<TJoinOp::type, TJoinOp::INNER_JOIN>,
std::integral_constant<TJoinOp::type, TJoinOp::LEFT_SEMI_JOIN>,
@@ -35,7 +57,20 @@ using JoinOpVariants =
std::integral_constant<TJoinOp::type, TJoinOp::RIGHT_SEMI_JOIN>,
std::integral_constant<TJoinOp::type, TJoinOp::RIGHT_ANTI_JOIN>,
std::integral_constant<TJoinOp::type, TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN>,
- std::integral_constant<TJoinOp::type, TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>>;
+ std::integral_constant<TJoinOp::type, TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>,
+ std::integral_constant<TJoinOp::type, TJoinOp::ASOF_LEFT_INNER_JOIN>,
+ std::integral_constant<TJoinOp::type, TJoinOp::ASOF_LEFT_OUTER_JOIN>>;
+
+inline bool is_asof_join(TJoinOp::type join_op) {
+ return join_op == TJoinOp::ASOF_LEFT_INNER_JOIN || join_op == TJoinOp::ASOF_LEFT_OUTER_JOIN;
+}
+
+template <int JoinOpType>
+inline constexpr bool is_asof_join_op_v =
+ JoinOpType == TJoinOp::ASOF_LEFT_INNER_JOIN || JoinOpType == TJoinOp::ASOF_LEFT_OUTER_JOIN;
+
+template <int JoinOpType>
+inline constexpr bool is_asof_outer_join_op_v = JoinOpType == TJoinOp::ASOF_LEFT_OUTER_JOIN;
template <class T>
using PrimaryTypeHashTableContext =
@@ -219,4 +254,99 @@ inline void try_convert_to_direct_mapping(
primary_to_direct_mapping(context, key_columns, variant_ptrs);
}
+// ASOF JOIN index with inline values for cache-friendly branchless binary search.
+// IntType is the integer representation of the ASOF column value:
+// uint32_t for DateV2, uint64_t for DateTimeV2 and TimestampTZ.
+// Rows are sorted by asof_value during build, then materialized into SoA arrays
+// so probe-side binary search only touches the ASOF values hot path.
+template <typename IntType>
+struct AsofIndexGroup {
+ using int_type = IntType;
+
+ struct Entry {
+ IntType asof_value;
+ uint32_t row_index; // 1-based, 0 = invalid/padding
+ };
+
+ std::vector<Entry> entries;
+ std::vector<IntType> asof_values;
+ std::vector<uint32_t> row_indexes;
+
+ void add_row(IntType value, uint32_t row_idx) { entries.push_back({value, row_idx}); }
+
+ void sort_and_finalize() {
+ if (entries.empty()) {
+ return;
+ }
+ if (entries.size() > 1) {
+ pdqsort(entries.begin(), entries.end(),
+ [](const Entry& a, const Entry& b) { return a.asof_value < b.asof_value; });
+ }
+
+ asof_values.resize(entries.size());
+ row_indexes.resize(entries.size());
+ for (size_t i = 0; i < entries.size(); ++i) {
+ asof_values[i] = entries[i].asof_value;
+ row_indexes[i] = entries[i].row_index;
+ }
+
+ std::vector<Entry>().swap(entries);
+ }
+
+ const IntType* values_data() const { return asof_values.data(); }
+
+ // Branchless lower_bound: first i where asof_values[i] >= target
+ ALWAYS_INLINE size_t lower_bound(IntType target) const {
+ size_t lo = 0, n = asof_values.size();
+ while (n > 1) {
+ size_t half = n / 2;
+ lo += half * (asof_values[lo + half] < target);
+ n -= half;
+ }
+ if (lo < asof_values.size()) {
+ lo += (asof_values[lo] < target);
+ }
+ return lo;
+ }
+
+ // Branchless upper_bound: first i where asof_values[i] > target
+ ALWAYS_INLINE size_t upper_bound(IntType target) const {
+ size_t lo = 0, n = asof_values.size();
+ while (n > 1) {
+ size_t half = n / 2;
+ lo += half * (asof_values[lo + half] <= target);
+ n -= half;
+ }
+ if (lo < asof_values.size()) {
+ lo += (asof_values[lo] <= target);
+ }
+ return lo;
+ }
+
+ // Semantics by (is_greater, is_strict):
+ // (true, false): probe >= build -> find largest build value <= probe
+ // (true, true): probe > build -> find largest build value < probe
+ // (false, false): probe <= build -> find smallest build value >= probe
+ // (false, true): probe < build -> find smallest build value > probe
+ // Returns the build row index of the best match, or 0 if no match.
+ template <bool IsGreater, bool IsStrict>
+ ALWAYS_INLINE uint32_t find_best_match(IntType probe_value) const {
+ if (asof_values.empty()) {
+ return 0;
+ }
+ if constexpr (IsGreater) {
+ size_t pos = IsStrict ? lower_bound(probe_value) : upper_bound(probe_value);
+ return pos > 0 ? row_indexes[pos - 1] : 0;
+ } else {
+ size_t pos = IsStrict ? upper_bound(probe_value) : lower_bound(probe_value);
+ return pos < asof_values.size() ? row_indexes[pos] : 0;
+ }
+ }
+};
+
+// Type-erased container for all ASOF index groups.
+// DateV2 -> uint32_t, DateTimeV2/TimestampTZ -> uint64_t.
+using AsofIndexVariant = std::variant<std::monostate, std::vector<AsofIndexGroup<uint32_t>>,
+ std::vector<AsofIndexGroup<uint64_t>>>;
+
} // namespace doris
diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
index 993149f0d36..511606c3d0b 100644
--- a/be/src/pipeline/dependency.h
+++ b/be/src/pipeline/dependency.h
@@ -631,6 +631,18 @@ struct HashJoinSharedState : public JoinSharedState {
// local filter will always be applied, and in filter could guarantee precise filtering
// ATTN: we should disable always_true logic for in filter when we set this flag
bool left_semi_direct_return = false;
+
+ // ASOF JOIN specific fields
+ // Whether the inequality is >= or > (true) vs <= or < (false)
+ bool asof_inequality_is_greater = true;
+ // Whether the inequality is strict (> or <) vs non-strict (>= or <=)
+ bool asof_inequality_is_strict = false;
+
+ // ASOF JOIN pre-sorted index with inline values for O(log K) branchless lookup
+ // Typed AsofIndexGroups stored in a variant (uint32_t for DateV2, uint64_t for DateTimeV2/TimestampTZ)
+ AsofIndexVariant asof_index_groups;
+ // build_row_index -> bucket_id for O(1) reverse lookup
+ std::vector<uint32_t> asof_build_row_to_bucket;
};
struct PartitionedHashJoinSharedState
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 91ba6f6781a..f6747c00319 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -88,6 +88,18 @@ Status HashJoinBuildSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo
_build_table_insert_timer = ADD_TIMER(record_profile, "BuildTableInsertTime");
_build_expr_call_timer = ADD_TIMER(record_profile, "BuildExprCallTime");
+ // ASOF index build counters (only for ASOF join types)
+ if (is_asof_join(p._join_op) && state->enable_profile() && state->profile_level() >= 2) {
+ static constexpr auto ASOF_INDEX_BUILD_TIMER = "AsofIndexBuildTime";
+ _asof_index_total_timer = ADD_TIMER_WITH_LEVEL(custom_profile(), ASOF_INDEX_BUILD_TIMER, 2);
+ _asof_index_expr_timer = ADD_CHILD_TIMER_WITH_LEVEL(custom_profile(), "AsofIndexExprTime",
+ ASOF_INDEX_BUILD_TIMER, 2);
+ _asof_index_sort_timer = ADD_CHILD_TIMER_WITH_LEVEL(custom_profile(), "AsofIndexSortTime",
+ ASOF_INDEX_BUILD_TIMER, 2);
+ _asof_index_group_timer = ADD_CHILD_TIMER_WITH_LEVEL(custom_profile(), "AsofIndexGroupTime",
+ ASOF_INDEX_BUILD_TIMER, 2);
+ }
+
_runtime_filter_producer_helper = std::make_shared<RuntimeFilterProducerHelper>(
_should_build_hash_table, p._is_broadcast_join);
RETURN_IF_ERROR(_runtime_filter_producer_helper->init(state, _build_expr_ctxs,
@@ -162,7 +174,8 @@ size_t HashJoinBuildSinkLocalState::get_reserve_mem_size(RuntimeState* state, bo
_build_side_mutable_block = vectorized::MutableBlock(std::move(block));
});
vectorized::ColumnUInt8::MutablePtr null_map_val;
- if (p._join_op == TJoinOp::LEFT_OUTER_JOIN || p._join_op == TJoinOp::FULL_OUTER_JOIN) {
+ if (p._join_op == TJoinOp::LEFT_OUTER_JOIN || p._join_op == TJoinOp::FULL_OUTER_JOIN ||
+ p._join_op == TJoinOp::ASOF_LEFT_OUTER_JOIN) {
converted_columns = _convert_block_to_null(block);
// first row is mocked
for (int i = 0; i < block.columns(); i++) {
@@ -286,7 +299,8 @@ void HashJoinBuildSinkLocalState::init_short_circuit_for_probe() {
(empty_block &&
(p._join_op == TJoinOp::INNER_JOIN || p._join_op == TJoinOp::LEFT_SEMI_JOIN ||
p._join_op == TJoinOp::RIGHT_OUTER_JOIN || p._join_op == TJoinOp::RIGHT_SEMI_JOIN ||
- p._join_op == TJoinOp::RIGHT_ANTI_JOIN))) &&
+ p._join_op == TJoinOp::RIGHT_ANTI_JOIN ||
+ p._join_op == TJoinOp::ASOF_LEFT_INNER_JOIN))) &&
!p._is_mark_join;
//when build table rows is 0 and not have other_join_conjunct and not _is_mark_join and join type is one of LEFT_OUTER_JOIN/FULL_OUTER_JOIN/LEFT_ANTI_JOIN
@@ -294,7 +308,181 @@ void HashJoinBuildSinkLocalState::init_short_circuit_for_probe() {
_shared_state->empty_right_table_need_probe_dispose =
(empty_block && !p._have_other_join_conjunct && !p._is_mark_join) &&
(p._join_op == TJoinOp::LEFT_OUTER_JOIN || p._join_op == TJoinOp::FULL_OUTER_JOIN ||
- p._join_op == TJoinOp::LEFT_ANTI_JOIN);
+ p._join_op == TJoinOp::LEFT_ANTI_JOIN || p._join_op == TJoinOp::ASOF_LEFT_OUTER_JOIN);
+}
+
+Status HashJoinBuildSinkLocalState::build_asof_index(vectorized::Block& block) {
+ auto& p = _parent->cast<HashJoinBuildSinkOperatorX>();
+
+ // Only for ASOF JOIN types
+ if (!is_asof_join(p._join_op)) {
+ return Status::OK();
+ }
+
+ SCOPED_TIMER(_asof_index_total_timer);
+
+ DORIS_CHECK(block.rows() != 0);
+ if (block.rows() == 1) {
+ // Empty or only mock row
+ return Status::OK();
+ }
+
+ // Get hash table's first and next arrays to traverse buckets
+ uint32_t bucket_size = 0;
+ const uint32_t* first_array = nullptr;
+ const uint32_t* next_array = nullptr;
+ size_t build_rows = 0;
+
+ std::visit(vectorized::Overload {[&](std::monostate&) {},
+ [&](auto&& hash_table_ctx) {
+ auto* hash_table = hash_table_ctx.hash_table.get();
+ DORIS_CHECK(hash_table);
+ bucket_size = hash_table->get_bucket_size();
+ first_array = hash_table->get_first().data();
+ next_array = hash_table->get_next().data();
+ build_rows = hash_table->size();
+ }},
+ _shared_state->hash_table_variant_vector.front()->method_variant);
+
+ if (bucket_size == 0) {
+ return Status::OK();
+ }
+ DORIS_CHECK(first_array);
+ DORIS_CHECK(next_array);
+
+ // Set inequality direction from opcode (moved from probe open())
+ _shared_state->asof_inequality_is_greater =
+ (p._asof_opcode == TExprOpcode::GE || p._asof_opcode == TExprOpcode::GT);
+ _shared_state->asof_inequality_is_strict =
+ (p._asof_opcode == TExprOpcode::GT || p._asof_opcode == TExprOpcode::LT);
+
+ // Compute build ASOF column by executing build-side expression directly on build block.
+ // Expression is prepared against build child's row_desc, matching the build block layout.
+ DORIS_CHECK(p._asof_build_side_expr);
+ int result_col_idx = -1;
+ {
+ SCOPED_TIMER(_asof_index_expr_timer);
+ RETURN_IF_ERROR(p._asof_build_side_expr->execute(&block, &result_col_idx));
+ }
+ DORIS_CHECK(result_col_idx >= 0 && result_col_idx < static_cast<int>(block.columns()));
+ auto asof_build_col =
+ block.get_by_position(result_col_idx).column->convert_to_full_column_if_const();
+
+ // Handle nullable: extract nested column for value access, keep nullable for null checks
+ const vectorized::ColumnNullable* nullable_col = nullptr;
+ vectorized::ColumnPtr build_col_nested = asof_build_col;
+ if (asof_build_col->is_nullable()) {
+ nullable_col = assert_cast<const vectorized::ColumnNullable*>(asof_build_col.get());
+ build_col_nested = nullable_col->get_nested_column_ptr();
+ }
+
+ // Initialize reverse mapping (all build rows, including NULL ASOF values)
+ _shared_state->asof_build_row_to_bucket.resize(build_rows + 1, 0);
+
+ // Dispatch on ASOF column type to create typed AsofIndexGroups with inline values.
+ // Sub-group by actual key equality within each hash bucket (hash collisions),
+ // extract integer representation of ASOF values, then sort by inline values.
+ asof_column_dispatch(build_col_nested.get(), [&](const auto* typed_col) {
+ using ColType = std::remove_const_t<std::remove_pointer_t<decltype(typed_col)>>;
+
+ if constexpr (std::is_same_v<ColType, vectorized::IColumn>) {
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "Unsupported ASOF column type for inline optimization");
+ } else {
+ using IntType = typename ColType::value_type::underlying_value;
+ const auto& col_data = typed_col->get_data();
+
+ auto& groups = _shared_state->asof_index_groups
+ .emplace<std::vector<AsofIndexGroup<IntType>>>();
+
+ std::visit(
+ vectorized::Overload {
+ [&](std::monostate&) {},
+ [&](auto&& hash_table_ctx) {
+ auto* hash_table = hash_table_ctx.hash_table.get();
+ DORIS_CHECK(hash_table);
+ const auto* build_keys = hash_table->get_build_keys();
+ using KeyType = std::remove_const_t<
+ std::remove_pointer_t<decltype(build_keys)>>;
+ uint32_t next_group_id = 0;
+
+ // Group rows by equality key within each hash bucket,
+ // then sort each group by ASOF value only (pure integer compare).
+ // This avoids the previous approach of sorting by (build_key, asof_value)
+ // which required memcmp per comparison.
+ //
+ // For each bucket: walk the chain, find-or-create a group for
+ // each distinct build_key, insert rows into the group with their
+ // inline ASOF value. After all buckets are processed, sort each group.
+
+ // Map from build_key -> group_id, reused across buckets.
+ // Within a single hash bucket, the number of distinct keys is
+ // typically very small (hash collisions are rare), so a flat
+ // scan is efficient.
+ struct KeyGroupEntry {
+ KeyType key;
+ uint32_t group_id;
+ };
+ std::vector<KeyGroupEntry> bucket_key_groups;
+
+ {
+ SCOPED_TIMER(_asof_index_group_timer);
+ for (uint32_t bucket = 0; bucket <= bucket_size; ++bucket) {
+ uint32_t row_idx = first_array[bucket];
+ if (row_idx == 0) {
+ continue;
+ }
+
+ // For each row in this bucket's chain, find-or-create its group
+ bucket_key_groups.clear();
+ while (row_idx != 0) {
+ DORIS_CHECK(row_idx <= build_rows);
+ const auto& key = build_keys[row_idx];
+
+ // Linear scan to find existing group for this key.
+ // Bucket chains are short (avg ~1-2 distinct keys per bucket),
+ // so this is faster than a hash map.
+ uint32_t group_id = UINT32_MAX;
+ for (const auto& entry : bucket_key_groups) {
+ if (entry.key == key) {
+ group_id = entry.group_id;
+ break;
+ }
+ }
+ if (group_id == UINT32_MAX) {
+ group_id = next_group_id++;
+ DORIS_CHECK(group_id == groups.size());
+ groups.emplace_back();
+ bucket_key_groups.push_back({key, group_id});
+ }
+
+ _shared_state->asof_build_row_to_bucket[row_idx] =
+ group_id;
+ if (!(nullable_col &&
+ nullable_col->is_null_at(row_idx))) {
+ groups[group_id].add_row(
+ col_data[row_idx].to_date_int_val(),
+ row_idx);
+ }
+
+ row_idx = next_array[row_idx];
+ }
+ }
+ }
+
+ // Sort each group by ASOF value only (pure integer comparison).
+ {
+ SCOPED_TIMER(_asof_index_sort_timer);
+ for (auto& group : groups) {
+ group.sort_and_finalize();
+ }
+ }
+ }},
+ _shared_state->hash_table_variant_vector.front()->method_variant);
+ }
+ });
+
+ return Status::OK();
}
Status HashJoinBuildSinkLocalState::_do_evaluate(vectorized::Block& block,
@@ -379,7 +567,8 @@ Status HashJoinBuildSinkLocalState::process_build_block(RuntimeState* state,
vectorized::ColumnRawPtrs raw_ptrs(_build_expr_ctxs.size());
vectorized::ColumnUInt8::MutablePtr null_map_val;
- if (p._join_op == TJoinOp::LEFT_OUTER_JOIN || p._join_op == TJoinOp::FULL_OUTER_JOIN) {
+ if (p._join_op == TJoinOp::LEFT_OUTER_JOIN || p._join_op == TJoinOp::FULL_OUTER_JOIN ||
+ p._join_op == TJoinOp::ASOF_LEFT_OUTER_JOIN) {
_convert_block_to_null(block);
// first row is mocked
for (int i = 0; i < block.columns(); i++) {
@@ -554,6 +743,23 @@ Status HashJoinBuildSinkOperatorX::init(const TPlanNode& tnode, RuntimeState* st
}
}
+ // For ASOF JOIN, extract the build-side expression from match_condition field.
+ // match_condition is bound on input tuples (left child output + right child output),
+ // so child(1) references build child's slots directly.
+ if (is_asof_join(_join_op)) {
+ DORIS_CHECK(tnode.hash_join_node.__isset.match_condition);
+ DORIS_CHECK(!_asof_build_side_expr);
+ vectorized::VExprContextSPtr full_conjunct;
+ RETURN_IF_ERROR(vectorized::VExpr::create_expr_tree(tnode.hash_join_node.match_condition,
+ full_conjunct));
+ DORIS_CHECK(full_conjunct);
+ DORIS_CHECK(full_conjunct->root());
+ DORIS_CHECK(full_conjunct->root()->get_num_children() == 2);
+ _asof_opcode = full_conjunct->root()->op();
+ auto right_child_expr = full_conjunct->root()->get_child(1);
+ _asof_build_side_expr = std::make_shared<vectorized::VExprContext>(right_child_expr);
+ }
+
return Status::OK();
}
@@ -576,6 +782,13 @@ Status HashJoinBuildSinkOperatorX::prepare(RuntimeState* state) {
};
init_keep_column_flags(row_desc().tuple_descriptors(), _should_keep_column_flags);
RETURN_IF_ERROR(vectorized::VExpr::prepare(_build_expr_ctxs, state, _child->row_desc()));
+ // Prepare ASOF build-side expression against build child's row_desc directly.
+ // match_condition is bound on input tuples, so child(1) references build child's slots.
+ if (is_asof_join(_join_op)) {
+ DORIS_CHECK(_asof_build_side_expr);
+ RETURN_IF_ERROR(_asof_build_side_expr->prepare(state, _child->row_desc()));
+ RETURN_IF_ERROR(_asof_build_side_expr->open(state));
+ }
return vectorized::VExpr::open(_build_expr_ctxs, state);
}
@@ -634,6 +847,8 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
RETURN_IF_ERROR(
local_state.process_build_block(state, (*local_state._shared_state->build_block)));
+ // For ASOF JOIN, build pre-sorted index for O(log K) lookup
+ RETURN_IF_ERROR(local_state.build_asof_index(*local_state._shared_state->build_block));
local_state.init_short_circuit_for_probe();
} else if (!local_state._should_build_hash_table) {
// the instance which is not build hash table, it's should wait the signal of hash table build finished.
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h
index f0cf73a82dd..3134965f263 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.h
+++ b/be/src/pipeline/exec/hashjoin_build_sink.h
@@ -39,6 +39,9 @@ public:
Status terminate(RuntimeState* state) override;
Status process_build_block(RuntimeState* state, vectorized::Block& block);
+ // Build ASOF JOIN pre-sorted index for O(log K) lookup
+ Status build_asof_index(vectorized::Block& block);
+
void init_short_circuit_for_probe();
bool build_unique() const;
@@ -96,6 +99,12 @@ protected:
RuntimeProfile::Counter* _build_blocks_memory_usage = nullptr;
RuntimeProfile::Counter* _hash_table_memory_usage = nullptr;
RuntimeProfile::Counter* _build_arena_memory_usage = nullptr;
+
+ // ASOF index build counters
+ RuntimeProfile::Counter* _asof_index_total_timer = nullptr;
+ RuntimeProfile::Counter* _asof_index_expr_timer = nullptr;
+ RuntimeProfile::Counter* _asof_index_sort_timer = nullptr;
+ RuntimeProfile::Counter* _asof_index_group_timer = nullptr;
};
class HashJoinBuildSinkOperatorX MOCK_REMOVE(final)
@@ -184,6 +193,11 @@ private:
// need to finalize variant column to speed up the join op
bool _need_finalize_variant_column = false;
+ // ASOF JOIN: build-side expression extracted from MATCH_CONDITION's right child
+ // Prepared against build child's row_desc directly (no intermediate tuple needed)
+ vectorized::VExprContextSPtr _asof_build_side_expr;
+ TExprOpcode::type _asof_opcode = TExprOpcode::INVALID_OPCODE;
+
bool _use_shared_hash_table = false;
std::atomic<bool> _signaled = false;
std::mutex _mutex;
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.cpp b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
index cb2c0df540c..2add03f640c 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.cpp
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.cpp
@@ -17,6 +17,7 @@
#include "hashjoin_probe_operator.h"
+#include <gen_cpp/Opcodes_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <string>
@@ -66,6 +67,18 @@ Status HashJoinProbeLocalState::init(RuntimeState* state, LocalStateInfo& info)
_non_equal_join_conjuncts_timer =
ADD_TIMER(custom_profile(), "NonEqualJoinConjunctEvaluationTime");
_init_probe_side_timer = ADD_TIMER(custom_profile(), "InitProbeSideTime");
+
+ // ASOF probe counters (only for ASOF join types)
+ if (is_asof_join(p._join_op) && state->enable_profile() && state->profile_level() >= 2) {
+ static constexpr auto SEARCH_HASH_TABLE_TIMER = "ProbeWhenSearchHashTableTime";
+ _asof_probe_expr_timer = ADD_CHILD_TIMER_WITH_LEVEL(custom_profile(), "AsofProbeExprTime",
+ SEARCH_HASH_TABLE_TIMER, 2);
+ _asof_probe_hash_chain_timer = ADD_CHILD_TIMER_WITH_LEVEL(
+ custom_profile(), "AsofProbeHashChainTime", SEARCH_HASH_TABLE_TIMER, 2);
+ _asof_probe_search_timer = ADD_CHILD_TIMER_WITH_LEVEL(
+ custom_profile(), "AsofProbeBinarySearchTime", SEARCH_HASH_TABLE_TIMER, 2);
+ }
+
return Status::OK();
}
@@ -75,6 +88,7 @@ Status HashJoinProbeLocalState::open(RuntimeState* state) {
RETURN_IF_ERROR(JoinProbeLocalState::open(state));
auto& p = _parent->cast<HashJoinProbeOperatorX>();
+
Status res;
std::visit(
[&](auto&& join_op_variants, auto have_other_join_conjunct) {
@@ -511,6 +525,20 @@ Status HashJoinProbeOperatorX::init(const TPlanNode& tnode, RuntimeState* state)
_join_op == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN);
}
+ // For ASOF JOIN, extract the probe-side expression from match_condition field.
+ // match_condition is bound on input tuples, so child(0) references probe child's slots.
+ if (is_asof_join(_join_op)) {
+ DORIS_CHECK(tnode.hash_join_node.__isset.match_condition);
+ vectorized::VExprContextSPtr full_conjunct;
+ RETURN_IF_ERROR(vectorized::VExpr::create_expr_tree(tnode.hash_join_node.match_condition,
+ full_conjunct));
+ DORIS_CHECK(full_conjunct);
+ DORIS_CHECK(full_conjunct->root());
+ DORIS_CHECK(full_conjunct->root()->get_num_children() == 2);
+ auto left_child_expr = full_conjunct->root()->get_child(0);
+ _asof_probe_expr = std::make_shared<vectorized::VExprContext>(left_child_expr);
+ }
+
return Status::OK();
}
@@ -550,6 +578,13 @@ Status HashJoinProbeOperatorX::prepare(RuntimeState* state) {
}
RETURN_IF_ERROR(vectorized::VExpr::prepare(_probe_expr_ctxs, state, _child->row_desc()));
+ // Prepare ASOF probe-side expression against probe child's row_desc directly.
+ if (is_asof_join(_join_op)) {
+ DORIS_CHECK(_asof_probe_expr);
+ RETURN_IF_ERROR(_asof_probe_expr->prepare(state, _child->row_desc()));
+ RETURN_IF_ERROR(_asof_probe_expr->open(state));
+ }
+
DCHECK(_build_side_child != nullptr);
// right table data types
_right_table_data_types =
@@ -617,11 +652,12 @@ Status HashJoinProbeOperatorX::prepare(RuntimeState* state) {
continue;
}
- /// For outer join(left/right/full), the non-nullable columns may be converted to nullable.
+ /// For outer join(left/right/full/asof), the non-nullable columns may be converted to nullable.
const auto accept_nullable_not_match =
_join_op == TJoinOp::FULL_OUTER_JOIN ||
- (slot_on_left ? _join_op == TJoinOp::RIGHT_OUTER_JOIN
- : _join_op == TJoinOp::LEFT_OUTER_JOIN);
+ (slot_on_left ? (_join_op == TJoinOp::RIGHT_OUTER_JOIN)
+ : (_join_op == TJoinOp::LEFT_OUTER_JOIN ||
+ _join_op == TJoinOp::ASOF_LEFT_OUTER_JOIN));
if (accept_nullable_not_match) {
auto data_type_non_nullable = vectorized::remove_nullable(data_type);
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h
index 6c5232b3720..c7f4b7d0c59 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.h
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.h
@@ -40,7 +40,9 @@ using HashTableCtxVariants =
ProcessHashTableProbe<TJoinOp::RIGHT_SEMI_JOIN>,
ProcessHashTableProbe<TJoinOp::RIGHT_ANTI_JOIN>,
ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN>,
- ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>>;
+ ProcessHashTableProbe<TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN>,
+ ProcessHashTableProbe<TJoinOp::ASOF_LEFT_INNER_JOIN>,
+ ProcessHashTableProbe<TJoinOp::ASOF_LEFT_OUTER_JOIN>>;
class HashJoinProbeOperatorX;
class HashJoinProbeLocalState MOCK_REMOVE(final)
@@ -78,6 +80,8 @@ private:
template <int JoinOpType>
friend struct ProcessHashTableProbe;
+ // one probe row may match multiple build rows. and we need output whenever we filled a block.
+ // so we use _probe_index to record the current probe row index for multi times do probe for one probe block.
int _probe_index = -1;
uint32_t _build_index = 0;
bool _ready_probe = false;
@@ -115,6 +119,11 @@ private:
RuntimeProfile::Counter* _init_probe_side_timer = nullptr;
RuntimeProfile::Counter* _build_side_output_timer = nullptr;
RuntimeProfile::Counter* _non_equal_join_conjuncts_timer = nullptr;
+
+ // ASOF probe counters
+ RuntimeProfile::Counter* _asof_probe_expr_timer = nullptr;
+ RuntimeProfile::Counter* _asof_probe_hash_chain_timer = nullptr;
+ RuntimeProfile::Counter* _asof_probe_search_timer = nullptr;
};
class HashJoinProbeOperatorX MOCK_REMOVE(final)
@@ -188,6 +197,10 @@ private:
// probe expr
vectorized::VExprContextSPtrs _probe_expr_ctxs;
+ // ASOF JOIN: probe-side expression extracted from MATCH_CONDITION's left child
+ // Prepared against probe child's row_desc directly
+ vectorized::VExprContextSPtr _asof_probe_expr;
+
vectorized::DataTypes _right_table_data_types;
vectorized::DataTypes _left_table_data_types;
std::vector<SlotId> _hash_output_slot_ids;
diff --git a/be/src/pipeline/exec/join/asof_left_inner_join_impl.cpp b/be/src/pipeline/exec/join/asof_left_inner_join_impl.cpp
new file mode 100644
index 00000000000..0bb8dd9ecf3
--- /dev/null
+++ b/be/src/pipeline/exec/join/asof_left_inner_join_impl.cpp
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/PlanNodes_types.h>
+
+#include "process_hash_table_probe_impl.h"
+
+namespace doris::pipeline {
+
+INSTANTIATION_FOR(TJoinOp::ASOF_LEFT_INNER_JOIN);
+
+}
diff --git a/be/src/pipeline/exec/join/asof_left_outer_join_impl.cpp b/be/src/pipeline/exec/join/asof_left_outer_join_impl.cpp
new file mode 100644
index 00000000000..30a36e6f7a0
--- /dev/null
+++ b/be/src/pipeline/exec/join/asof_left_outer_join_impl.cpp
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/PlanNodes_types.h>
+
+#include "process_hash_table_probe_impl.h"
+
+namespace doris::pipeline {
+
+INSTANTIATION_FOR(TJoinOp::ASOF_LEFT_OUTER_JOIN);
+
+}
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe.h b/be/src/pipeline/exec/join/process_hash_table_probe.h
index 52dcc6415fa..fe18e17e844 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe.h
@@ -72,6 +72,12 @@ struct ProcessHashTableProbe {
Status do_mark_join_conjuncts(vectorized::Block* output_block, const uint8_t* null_map);
+ // ASOF JOIN optimized: use pre-sorted index to find best match in O(log K) per probe row
+ // Returns the number of matched rows
+ template <typename HashTableType>
+ uint32_t _find_batch_asof_optimized(HashTableType& hash_table_ctx, const uint8_t* null_map,
+ uint32_t probe_rows);
+
Status finalize_block_with_filter(vectorized::Block* output_block, size_t filter_column_id,
size_t column_to_keep);
@@ -126,6 +132,11 @@ struct ProcessHashTableProbe {
RuntimeProfile::Counter* _probe_side_output_timer = nullptr;
RuntimeProfile::Counter* _finish_probe_phase_timer = nullptr;
+ // ASOF probe counters
+ RuntimeProfile::Counter* _asof_probe_expr_timer = nullptr;
+ RuntimeProfile::Counter* _asof_probe_hash_chain_timer = nullptr;
+ RuntimeProfile::Counter* _asof_probe_search_timer = nullptr;
+
// See `HashJoinProbeOperatorX::_right_col_idx`
const size_t _right_col_idx;
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
index 85969391725..131e6f953b6 100644
--- a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
+++ b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
@@ -80,6 +80,9 @@ ProcessHashTableProbe<JoinOpType>::ProcessHashTableProbe(HashJoinProbeLocalState
_build_side_output_timer(parent->_build_side_output_timer),
_probe_side_output_timer(parent->_probe_side_output_timer),
_finish_probe_phase_timer(parent->_finish_probe_phase_timer),
+ _asof_probe_expr_timer(parent->_asof_probe_expr_timer),
+ _asof_probe_hash_chain_timer(parent->_asof_probe_hash_chain_timer),
+ _asof_probe_search_timer(parent->_asof_probe_search_timer),
_right_col_idx(_parent_operator->_right_col_idx),
_right_col_len(_parent_operator->_right_table_data_types.size()) {
constexpr int CALCULATE_ALL_MATCH_ONE_THRESHOLD = 2;
@@ -120,10 +123,14 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(vectorized::Mut
}
}
+ // For ASOF JOIN optimized path, skip lazy materialization check since we already found best match
+ constexpr bool is_asof_join = is_asof_join_op_v<JoinOpType>;
for (int i = 0; i < _right_col_len && i + _right_col_idx < mcol.size(); i++) {
const auto& column = *_build_block->safe_get_by_position(i).column;
- if (_right_output_slot_flags[i] &&
- !_parent_operator->is_lazy_materialized_column(i + (int)_right_col_idx)) {
+ bool should_output = _right_output_slot_flags[i] &&
+ (is_asof_join || !_parent_operator->is_lazy_materialized_column(
+ i + (int)_right_col_idx));
+ if (should_output) {
if (!build_index_has_zero && _build_column_has_null[i]) {
assert_cast<vectorized::ColumnNullable*>(mcol[i + _right_col_idx].get())
->insert_indices_from_not_has_null(column, _build_indexs.get_data().data(),
@@ -165,7 +172,11 @@ void ProcessHashTableProbe<JoinOpType>::probe_side_output_column(vectorized::Mut
}
}
- if (_left_output_slot_flags[i] && !_parent_operator->is_lazy_materialized_column(i)) {
+ // For ASOF JOIN optimized path, skip lazy materialization check
+ constexpr bool is_asof_join = is_asof_join_op_v<JoinOpType>;
+ bool should_output = _left_output_slot_flags[i] &&
+ (is_asof_join || !_parent_operator->is_lazy_materialized_column(i));
+ if (should_output) {
auto& column = probe_block.get_by_position(i).column;
insert_with_indexs(mcol[i], column, _probe_indexs.get_data(), all_match_one);
} else {
@@ -206,6 +217,248 @@ typename HashTableType::State ProcessHashTableProbe<JoinOpType>::_init_probe_sid
return typename HashTableType::State(_parent->_probe_columns);
}
+// ASOF JOIN optimized: two-phase probe with prefetch and compile-time dispatch.
+// Phase 1: Batch hash chain walk — resolve all probe rows to matching build rows.
+// Phase 2: Batch binary search — find ASOF best match for each resolved row.
+// This separation improves cache locality since phase 2 accesses AsofIndexGroup
+// data sequentially without interleaving hash chain walks.
+template <int JoinOpType>
+template <typename HashTableType>
+uint32_t ProcessHashTableProbe<JoinOpType>::
+ _find_batch_asof_optimized( // NOLINT(readability-function-cognitive-complexity)
+ HashTableType& hash_table_ctx, const uint8_t* null_map, uint32_t probe_rows) {
+ auto* shared_state = _parent->_shared_state;
+ constexpr bool is_outer_join = is_asof_outer_join_op_v<JoinOpType>;
+ auto& probe_index = _parent->_probe_index;
+
+ // asof_index_groups only populated by build_asof_index when build block has real data
+ if (std::holds_alternative<std::monostate>(shared_state->asof_index_groups)) {
+ if constexpr (is_outer_join) {
+ uint32_t matched_cnt = 0;
+ for (; probe_index < probe_rows && matched_cnt < _batch_size; ++probe_index) {
+ _probe_indexs.get_element(matched_cnt) = probe_index;
+ _build_indexs.get_element(matched_cnt) = 0;
+ matched_cnt++;
+ }
+ return matched_cnt;
+ }
+ probe_index = probe_rows;
+ return 0;
+ }
+
+ // Get ASOF probe expression from operator and execute on probe block.
+ auto& asof_probe_expr = _parent_operator->_asof_probe_expr;
+ DORIS_CHECK(asof_probe_expr);
+
+ auto& probe_block = _parent->_probe_block;
+ int probe_col_idx = -1;
+ {
+ SCOPED_TIMER(_asof_probe_expr_timer);
+ auto st = asof_probe_expr->execute(&probe_block, &probe_col_idx);
+ DORIS_CHECK(st.ok());
+ }
+ DORIS_CHECK(probe_col_idx >= 0 && probe_col_idx < static_cast<int>(probe_block.columns()));
+ auto probe_col_ptr =
+ probe_block.get_by_position(probe_col_idx).column->convert_to_full_column_if_const();
+
+ // Remove nullable wrapper for comparison - keep original for null check
+ vectorized::ColumnPtr probe_col_for_compare = probe_col_ptr;
+ const uint8_t* asof_probe_null_map = nullptr;
+ if (probe_col_ptr->is_nullable()) {
+ const auto* nullable_probe_col =
+ assert_cast<const vectorized::ColumnNullable*>(probe_col_ptr.get());
+ asof_probe_null_map = nullable_probe_col->get_null_map_data().data();
+ probe_col_for_compare = nullable_probe_col->get_nested_column_ptr();
+ }
+ const auto* probe_col = probe_col_for_compare.get();
+
+ bool is_greater = shared_state->asof_inequality_is_greater;
+ bool is_strict = shared_state->asof_inequality_is_strict;
+
+ // Two-phase probe with compile-time dispatched binary search
+ [[maybe_unused]] auto probe_with_index = [&](const auto& asof_groups,
+ const auto* typed_probe_col) -> uint32_t {
+ using IntType =
+ typename std::remove_reference_t<decltype(asof_groups)>::value_type::int_type;
+ const auto& probe_data = typed_probe_col->get_data();
+
+ const auto& next_arr = hash_table_ctx.hash_table->get_next();
+ const auto* build_keys = hash_table_ctx.hash_table->get_build_keys();
+ const auto* bucket_row_to_group = shared_state->asof_build_row_to_bucket.data();
+
+ // Determine the batch range: [probe_index, batch_end)
+ uint32_t batch_end = std::min(probe_rows, probe_index + static_cast<uint32_t>(_batch_size));
+ uint32_t batch_start = probe_index;
+ uint32_t batch_count = batch_end - batch_start;
+
+ // Temporary arrays for two-phase processing.
+ // resolved_group_ids[i]: resolved ASOF group id + 1 for probe row (batch_start+i),
+ // 0 means no match or NULL probe key.
+ // We use stack allocation for small batches, heap for large.
+ constexpr uint32_t STACK_LIMIT = 4096;
+ uint32_t stack_buf[STACK_LIMIT];
+ IntType stack_probe_value_buf[STACK_LIMIT];
+ std::unique_ptr<uint32_t[]> heap_buf;
+ std::unique_ptr<IntType[]> heap_probe_value_buf;
+ uint32_t* resolved_group_ids;
+ IntType* probe_values;
+ if (batch_count <= STACK_LIMIT) {
+ resolved_group_ids = stack_buf;
+ probe_values = stack_probe_value_buf;
+ } else {
+ heap_buf = std::make_unique<uint32_t[]>(batch_count);
+ heap_probe_value_buf = std::make_unique<IntType[]>(batch_count);
+ resolved_group_ids = heap_buf.get();
+ probe_values = heap_probe_value_buf.get();
+ }
+
+ // ============================================================
+ // Phase 1: Batch hash chain walk
+ // ============================================================
+ {
+ SCOPED_TIMER(_asof_probe_hash_chain_timer);
+ constexpr int PREFETCH_AHEAD = 4;
+
+ for (uint32_t i = 0; i < batch_count; ++i) {
+ uint32_t pi = batch_start + i;
+
+ // Prefetch: look ahead at the first[] entry for a future probe row.
+ if (i + PREFETCH_AHEAD < batch_count) {
+ uint32_t future_pi = batch_start + i + PREFETCH_AHEAD;
+ uint32_t future_bucket = hash_table_ctx.bucket_nums[future_pi];
+ __builtin_prefetch(&next_arr[future_bucket], 0, 1);
+ }
+
+ // Skip NULL probe keys
+ if ((null_map && null_map[pi]) ||
+ (asof_probe_null_map && asof_probe_null_map[pi])) {
+ resolved_group_ids[i] = 0;
+ continue;
+ }
+
+ // Walk hash chain
+ uint32_t row = hash_table_ctx.bucket_nums[pi];
+ uint32_t match = 0;
+ while (row != 0) {
+ DORIS_CHECK(row < next_arr.size());
+ if (build_keys[row] == hash_table_ctx.keys[pi]) {
+ match = row;
+ break;
+ }
+ row = next_arr[row];
+ }
+ if (match == 0) {
+ resolved_group_ids[i] = 0;
+ continue;
+ }
+
+ DORIS_CHECK(match < shared_state->asof_build_row_to_bucket.size());
+ uint32_t group_id = bucket_row_to_group[match];
+ DORIS_CHECK(group_id < asof_groups.size());
+ resolved_group_ids[i] = group_id + 1;
+ probe_values[i] = static_cast<IntType>(probe_data[pi].to_date_int_val());
+ }
+ }
+
+ // ============================================================
+ // Phase 2: Batch binary search with compile-time dispatch
+ // ============================================================
+ // Dispatch is_greater/is_strict at compile time to eliminate branches in the hot loop.
+ auto do_binary_search = [&]<bool IsGreater, bool IsStrict>() -> uint32_t {
+ SCOPED_TIMER(_asof_probe_search_timer);
+ constexpr int PREFETCH_AHEAD = 4;
+ uint32_t cnt = 0;
+
+ for (uint32_t i = 0; i < batch_count; ++i) {
+ uint32_t pi = batch_start + i;
+ uint32_t encoded_group_id = resolved_group_ids[i];
+
+ if (encoded_group_id == 0) {
+ if constexpr (is_outer_join) {
+ _probe_indexs.get_element(cnt) = pi;
+ _build_indexs.get_element(cnt) = 0;
+ cnt++;
+ }
+ continue;
+ }
+ uint32_t group_id = encoded_group_id - 1;
+
+ // Prefetch the ASOF value array of a future group
+ if (i + PREFETCH_AHEAD < batch_count) {
+ uint32_t future_group_id = resolved_group_ids[i + PREFETCH_AHEAD];
+ if (future_group_id != 0) {
+ uint32_t future_gid = future_group_id - 1;
+ __builtin_prefetch(asof_groups[future_gid].values_data(), 0, 1);
+ }
+ }
+
+ uint32_t best_build_row =
+ asof_groups[group_id].template find_best_match<IsGreater, IsStrict>(
+ probe_values[i]);
+
+ if (best_build_row != 0) {
+ _probe_indexs.get_element(cnt) = pi;
+ _build_indexs.get_element(cnt) = best_build_row;
+ cnt++;
+ } else if constexpr (is_outer_join) {
+ _probe_indexs.get_element(cnt) = pi;
+ _build_indexs.get_element(cnt) = 0;
+ cnt++;
+ }
+ }
+ return cnt;
+ };
+
+ // 4-way compile-time dispatch on (is_greater, is_strict)
+ uint32_t cnt;
+ if (is_greater) {
+ if (is_strict) {
+ cnt = do_binary_search.template operator()<true, true>();
+ } else {
+ cnt = do_binary_search.template operator()<true, false>();
+ }
+ } else {
+ if (is_strict) {
+ cnt = do_binary_search.template operator()<false, true>();
+ } else {
+ cnt = do_binary_search.template operator()<false, false>();
+ }
+ }
+
+ // Advance probe_index past the entire batch
+ probe_index = batch_end;
+ return cnt;
+ };
+
+ // Dispatch on AsofIndexVariant to get the correct integer type,
+ // then cast probe column to the matching concrete type
+ uint32_t matched_cnt = std::visit(
+ vectorized::Overload {
+ [](std::monostate&) -> uint32_t {
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "AsofIndexVariant should not be monostate here");
+ },
+ [&](std::vector<AsofIndexGroup<uint32_t>>& groups) -> uint32_t {
+ // DateV2
+ return probe_with_index(
+ groups, assert_cast<const vectorized::ColumnDateV2*>(probe_col));
+ },
+ [&](std::vector<AsofIndexGroup<uint64_t>>& groups) -> uint32_t {
+ // DateTimeV2 or TimestampTZ
+ if (const auto* c =
+ vectorized::check_and_get_column<vectorized::ColumnDateTimeV2>(
+ probe_col)) {
+ return probe_with_index(groups, c);
+ }
+ return probe_with_index(
+ groups,
+ assert_cast<const vectorized::ColumnTimeStampTz*>(probe_col));
+ }},
+ shared_state->asof_index_groups);
+
+ return matched_cnt;
+}
+
template <int JoinOpType>
template <typename HashTableType>
void ProcessHashTableProbe<JoinOpType>::process_direct_return(
@@ -243,9 +496,15 @@ Status ProcessHashTableProbe<JoinOpType>::process(HashTableType& hash_table_ctx,
auto& mcol = mutable_block.mutable_columns();
uint32_t current_offset = 0;
- if ((JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
- JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) &&
- _have_other_join_conjunct) {
+
+ // ASOF JOIN: always use optimized path with O(log K) binary search
+ constexpr bool is_asof_join = is_asof_join_op_v<JoinOpType>;
+ if constexpr (is_asof_join) {
+ SCOPED_TIMER(_search_hashtable_timer);
+ current_offset = _find_batch_asof_optimized(hash_table_ctx, null_map, probe_rows);
+ } else if ((JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
+ JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) &&
+ _have_other_join_conjunct) {
SCOPED_TIMER(_search_hashtable_timer);
/// If `_build_index_for_null_probe_key` is not zero, it means we are in progress of handling probe null key.
@@ -311,6 +570,11 @@ Status ProcessHashTableProbe<JoinOpType>::process(HashTableType& hash_table_ctx,
DCHECK_EQ(current_offset, output_block->rows());
COUNTER_UPDATE(_parent->_intermediate_rows_counter, current_offset);
+ // For ASOF JOIN, skip conjuncts filtering since we already found best match
+ if constexpr (is_asof_join) {
+ return Status::OK();
+ }
+
if (is_mark_join) {
bool ignore_null_map =
(JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
@@ -722,6 +986,7 @@ Status ProcessHashTableProbe<JoinOpType>::finish_probing(HashTableType& hash_tab
bool is_mark_join) {
SCOPED_TIMER(_finish_probe_phase_timer);
auto& mcol = mutable_block.mutable_columns();
+
if (is_mark_join) {
std::unique_ptr<vectorized::ColumnFilterHelper> mark_column =
std::make_unique<vectorized::ColumnFilterHelper>(*mcol[mcol.size() - 1]);
@@ -737,7 +1002,8 @@ Status ProcessHashTableProbe<JoinOpType>::finish_probing(HashTableType& hash_tab
if (block_size) {
if (mcol.size() < _right_col_len + _right_col_idx) {
return Status::InternalError(
- "output block invalid, mcol.size()={}, _right_col_len={}, _right_col_idx={}",
+ "output block invalid, mcol.size()={}, _right_col_len={}, "
+ "_right_col_idx={}",
mcol.size(), _right_col_len, _right_col_idx);
}
for (size_t j = 0; j < _right_col_len; ++j) {
diff --git a/be/src/pipeline/exec/join_build_sink_operator.cpp b/be/src/pipeline/exec/join_build_sink_operator.cpp
index b8ef52aaa45..fb9cc718232 100644
--- a/be/src/pipeline/exec/join_build_sink_operator.cpp
+++ b/be/src/pipeline/exec/join_build_sink_operator.cpp
@@ -49,7 +49,8 @@ JoinBuildSinkOperatorX<LocalStateType>::JoinBuildSinkOperatorX(ObjectPool* pool,
!tnode.hash_join_node.other_join_conjuncts.empty()) ||
tnode.hash_join_node.__isset.vother_join_conjunct)),
_match_all_probe(_join_op == TJoinOp::LEFT_OUTER_JOIN ||
- _join_op == TJoinOp::FULL_OUTER_JOIN),
+ _join_op == TJoinOp::FULL_OUTER_JOIN ||
+ _join_op == TJoinOp::ASOF_LEFT_OUTER_JOIN),
_match_all_build(_join_op == TJoinOp::RIGHT_OUTER_JOIN ||
_join_op == TJoinOp::FULL_OUTER_JOIN),
_build_unique(!_have_other_join_conjunct &&
@@ -96,7 +97,9 @@ JoinBuildSinkOperatorX<LocalStateType>::JoinBuildSinkOperatorX(ObjectPool* pool,
M(RIGHT_SEMI_JOIN) \
M(RIGHT_ANTI_JOIN) \
M(NULL_AWARE_LEFT_ANTI_JOIN) \
- M(NULL_AWARE_LEFT_SEMI_JOIN)
+ M(NULL_AWARE_LEFT_SEMI_JOIN) \
+ M(ASOF_LEFT_INNER_JOIN) \
+ M(ASOF_LEFT_OUTER_JOIN)
template <typename LocalStateType>
void JoinBuildSinkOperatorX<LocalStateType>::_init_join_op() {
diff --git a/be/src/pipeline/exec/join_probe_operator.cpp b/be/src/pipeline/exec/join_probe_operator.cpp
index 8d580141774..1d25460c04e 100644
--- a/be/src/pipeline/exec/join_probe_operator.cpp
+++ b/be/src/pipeline/exec/join_probe_operator.cpp
@@ -95,7 +95,8 @@ JoinProbeOperatorX<LocalStateType>::JoinProbeOperatorX(ObjectPool* pool, const T
!tnode.hash_join_node.other_join_conjuncts.empty()) ||
tnode.hash_join_node.__isset.vother_join_conjunct)),
_match_all_probe(_join_op == TJoinOp::LEFT_OUTER_JOIN ||
- _join_op == TJoinOp::FULL_OUTER_JOIN),
+ _join_op == TJoinOp::FULL_OUTER_JOIN ||
+ _join_op == TJoinOp::ASOF_LEFT_OUTER_JOIN),
_match_all_build(_join_op == TJoinOp::RIGHT_OUTER_JOIN ||
_join_op == TJoinOp::FULL_OUTER_JOIN),
_build_unique(!_have_other_join_conjunct &&
diff --git a/be/src/vec/common/hash_table/join_hash_table.h b/be/src/vec/common/hash_table/join_hash_table.h
index 9426829e056..c116a17777c 100644
--- a/be/src/vec/common/hash_table/join_hash_table.h
+++ b/be/src/vec/common/hash_table/join_hash_table.h
@@ -78,6 +78,12 @@ public:
DorisVector<uint8_t>& get_visited() { return visited; }
+ const DorisVector<uint32_t>& get_first() const { return first; }
+
+ const DorisVector<uint32_t>& get_next() const { return next; }
+
+ const Key* get_build_keys() const { return build_keys; }
+
bool empty_build_side() const { return _empty_build_side; }
void build(const Key* __restrict keys, const uint32_t* __restrict bucket_nums,
@@ -138,6 +144,14 @@ public:
build_idx, probe_rows, probe_idxs,
probe_visited, build_idxs);
}
+ // ASOF JOIN: for each probe row, find one matching build row (the closest match)
+ // The actual closest match logic is handled in ProcessHashTableProbe
+ if (JoinOpType == TJoinOp::ASOF_LEFT_INNER_JOIN ||
+ JoinOpType == TJoinOp::ASOF_LEFT_OUTER_JOIN) {
+ // Use conjunct path to get all matching rows, then filter in ProcessHashTableProbe
+ return _find_batch_conjunct<JoinOpType, false>(
+ keys, build_idx_map, probe_idx, build_idx, probe_rows, probe_idxs, build_idxs);
+ }
if (JoinOpType == TJoinOp::LEFT_ANTI_JOIN || JoinOpType == TJoinOp::LEFT_SEMI_JOIN ||
JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN) {
if (null_map) {
@@ -332,7 +346,8 @@ private:
JoinOpType == TJoinOp::LEFT_SEMI_JOIN ||
JoinOpType == TJoinOp::LEFT_ANTI_JOIN ||
JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN ||
- JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) {
+ JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN ||
+ JoinOpType == TJoinOp::ASOF_LEFT_OUTER_JOIN) {
// may over batch_size when emplace 0 into build_idxs
if (!build_idx) {
probe_idxs[matched_cnt] = probe_idx;
diff --git a/be/test/pipeline/operator/asof_join_test.cpp b/be/test/pipeline/operator/asof_join_test.cpp
new file mode 100644
index 00000000000..8f4a450db75
--- /dev/null
+++ b/be/test/pipeline/operator/asof_join_test.cpp
@@ -0,0 +1,387 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <initializer_list>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "pipeline/common/join_utils.h"
+#include "vec/columns/column_vector.h"
+#include "vec/runtime/timestamptz_value.h"
+
+namespace doris {
+using namespace vectorized;
+static ColumnPtr make_int32_column(const std::vector<int32_t>& values) {
+ auto col = ColumnInt32::create();
+ for (auto v : values) {
+ col->insert_value(v);
+ }
+ return col;
+}
+
+static ColumnPtr make_datev2_column(const std::vector<uint32_t>& values) {
+ auto col = ColumnDateV2::create();
+ for (auto v : values) {
+ col->insert_value(v);
+ }
+ return col;
+}
+
+template <typename IntType>
+static AsofIndexGroup<IntType> make_group(
+ std::initializer_list<std::pair<IntType, uint32_t>> rows) {
+ AsofIndexGroup<IntType> group;
+ for (const auto& [value, row_idx] : rows) {
+ group.add_row(value, row_idx);
+ }
+ group.sort_and_finalize();
+ return group;
+}
+
+template <typename IntType>
+static void expect_sorted(const std::vector<IntType>& values) {
+ for (size_t i = 1; i < values.size(); ++i) {
+ EXPECT_LE(values[i - 1], values[i]);
+ }
+}
+
+static bool contains_row(uint32_t row_idx, std::initializer_list<uint32_t> candidates) {
+ for (uint32_t candidate : candidates) {
+ if (candidate == row_idx) {
+ return true;
+ }
+ }
+ return false;
+}
+
+class AsofIndexGroupFinalizeTest : public ::testing::Test {};
+
+TEST_F(AsofIndexGroupFinalizeTest, EmptyGroup) {
+ AsofIndexGroup<uint32_t> group;
+ group.sort_and_finalize();
+ EXPECT_TRUE(group.entries.empty());
+ EXPECT_TRUE(group.asof_values.empty());
+ EXPECT_TRUE(group.row_indexes.empty());
+}
+
+TEST_F(AsofIndexGroupFinalizeTest, SortAndFinalizeBuildsSoA) {
+ auto group = make_group<uint32_t>({{40, 1}, {10, 2}, {30, 3}, {20, 4}, {50, 5}});
+ EXPECT_TRUE(group.entries.empty());
+ EXPECT_EQ(group.asof_values, (std::vector<uint32_t> {10, 20, 30, 40, 50}));
+ EXPECT_EQ(group.row_indexes, (std::vector<uint32_t> {2, 4, 3, 1, 5}));
+ EXPECT_EQ(group.values_data(), group.asof_values.data());
+}
+
+TEST_F(AsofIndexGroupFinalizeTest, SortAndFinalizeRetainsValueRowAlignment) {
+ std::vector<std::pair<uint32_t, uint32_t>> rows = {{30, 7}, {10, 2}, {30, 5}, {20, 9}, {10, 4}};
+ AsofIndexGroup<uint32_t> group;
+ for (const auto& [value, row_idx] : rows) {
+ group.add_row(value, row_idx);
+ }
+ group.sort_and_finalize();
+
+ EXPECT_TRUE(group.entries.empty());
+ expect_sorted(group.asof_values);
+ ASSERT_EQ(group.asof_values.size(), rows.size());
+ ASSERT_EQ(group.row_indexes.size(), rows.size());
+ for (size_t i = 0; i < group.row_indexes.size(); ++i) {
+ bool found = false;
+ for (const auto& [value, row_idx] : rows) {
+ if (row_idx == group.row_indexes[i]) {
+ EXPECT_EQ(value, group.asof_values[i]);
+ found = true;
+ break;
+ }
+ }
+ EXPECT_TRUE(found);
+ }
+}
+
+TEST_F(AsofIndexGroupFinalizeTest, SingleElementSkipsPdqsort) {
+ auto group = make_group<uint32_t>({{42, 7}});
+ EXPECT_TRUE(group.entries.empty());
+ ASSERT_EQ(group.asof_values.size(), 1U);
+ EXPECT_EQ(group.asof_values[0], 42U);
+ EXPECT_EQ(group.row_indexes[0], 7U);
+}
+
+TEST_F(AsofIndexGroupFinalizeTest, DoubleFinalizeIsIdempotent) {
+ auto group = make_group<uint32_t>({{20, 2}, {10, 1}});
+ EXPECT_EQ(group.asof_values, (std::vector<uint32_t> {10, 20}));
+ group.sort_and_finalize();
+ EXPECT_EQ(group.asof_values, (std::vector<uint32_t> {10, 20}));
+ EXPECT_EQ(group.row_indexes, (std::vector<uint32_t> {1, 2}));
+}
+
+TEST_F(AsofIndexGroupFinalizeTest, UInt64GroupsAreSupported) {
+ auto group =
+ make_group<uint64_t>({{30000000003ULL, 3}, {10000000001ULL, 1}, {20000000002ULL, 2}});
+ EXPECT_EQ(group.asof_values,
+ (std::vector<uint64_t> {10000000001ULL, 20000000002ULL, 30000000003ULL}));
+ EXPECT_EQ(group.row_indexes, (std::vector<uint32_t> {1, 2, 3}));
+}
+
+class AsofIndexGroupBoundTest : public ::testing::Test {};
+
+TEST_F(AsofIndexGroupBoundTest, EmptyGroupBoundReturnsZero) {
+ AsofIndexGroup<uint32_t> group;
+ group.sort_and_finalize();
+ EXPECT_EQ(group.lower_bound(42), 0U);
+ EXPECT_EQ(group.upper_bound(42), 0U);
+}
+
+TEST_F(AsofIndexGroupBoundTest, UInt64LowerAndUpperBound) {
+ auto group = make_group<uint64_t>(
+ {{10000000001ULL, 1}, {20000000002ULL, 2}, {20000000002ULL, 3}, {40000000004ULL, 4}});
+ EXPECT_EQ(group.lower_bound(5000000000ULL), 0U);
+ EXPECT_EQ(group.upper_bound(5000000000ULL), 0U);
+ EXPECT_EQ(group.lower_bound(20000000002ULL), 1U);
+ EXPECT_EQ(group.upper_bound(20000000002ULL), 3U);
+ EXPECT_EQ(group.lower_bound(30000000000ULL), 3U);
+ EXPECT_EQ(group.upper_bound(30000000000ULL), 3U);
+ EXPECT_EQ(group.lower_bound(50000000000ULL), 4U);
+ EXPECT_EQ(group.upper_bound(50000000000ULL), 4U);
+}
+
+TEST_F(AsofIndexGroupBoundTest, LowerAndUpperBoundRespectDuplicates) {
+ auto group = make_group<uint32_t>({{10, 1}, {20, 2}, {20, 3}, {40, 4}});
+ EXPECT_EQ(group.lower_bound(5), 0U);
+ EXPECT_EQ(group.upper_bound(5), 0U);
+ EXPECT_EQ(group.lower_bound(20), 1U);
+ EXPECT_EQ(group.upper_bound(20), 3U);
+ EXPECT_EQ(group.lower_bound(30), 3U);
+ EXPECT_EQ(group.upper_bound(30), 3U);
+ EXPECT_EQ(group.lower_bound(50), 4U);
+ EXPECT_EQ(group.upper_bound(50), 4U);
+}
+
+class AsofIndexGroupMatchTest : public ::testing::Test {
+protected:
+ AsofIndexGroup<uint32_t> group;
+
+ void SetUp() override {
+ group = make_group<uint32_t>({{10, 1}, {20, 2}, {30, 3}, {40, 4}, {50, 5}});
+ }
+};
+
+TEST_F(AsofIndexGroupMatchTest, FourInequalitiesAcrossRepresentativeValues) {
+ struct MatchCase {
+ uint32_t probe;
+ uint32_t ge;
+ uint32_t gt;
+ uint32_t le;
+ uint32_t lt;
+ };
+ std::vector<MatchCase> cases = {{5, 0, 0, 1, 1}, {10, 1, 0, 1, 2}, {25, 2, 2, 3, 3},
+ {30, 3, 2, 3, 4}, {50, 5, 4, 5, 0}, {100, 5, 5, 0, 0}};
+
+ for (const auto& test_case : cases) {
+ EXPECT_EQ((group.template find_best_match<true, false>(test_case.probe)), test_case.ge);
+ EXPECT_EQ((group.template find_best_match<true, true>(test_case.probe)), test_case.gt);
+ EXPECT_EQ((group.template find_best_match<false, false>(test_case.probe)), test_case.le);
+ EXPECT_EQ((group.template find_best_match<false, true>(test_case.probe)), test_case.lt);
+ }
+}
+
+TEST_F(AsofIndexGroupMatchTest, SingleElementGroupAcrossAllModes) {
+ auto single = make_group<uint32_t>({{30, 3}});
+ EXPECT_EQ((single.template find_best_match<true, false>(30)), 3U);
+ EXPECT_EQ((single.template find_best_match<true, false>(40)), 3U);
+ EXPECT_EQ((single.template find_best_match<true, false>(20)), 0U);
+ EXPECT_EQ((single.template find_best_match<true, true>(30)), 0U);
+ EXPECT_EQ((single.template find_best_match<true, true>(40)), 3U);
+ EXPECT_EQ((single.template find_best_match<false, false>(30)), 3U);
+ EXPECT_EQ((single.template find_best_match<false, false>(20)), 3U);
+ EXPECT_EQ((single.template find_best_match<false, false>(40)), 0U);
+ EXPECT_EQ((single.template find_best_match<false, true>(30)), 0U);
+ EXPECT_EQ((single.template find_best_match<false, true>(20)), 3U);
+ EXPECT_EQ((single.template find_best_match<false, true>(40)), 0U);
+}
+
+TEST_F(AsofIndexGroupMatchTest, AllEqualValuesAcrossAllModes) {
+ auto equal_group = make_group<uint32_t>({{30, 1}, {30, 2}, {30, 3}});
+
+ uint32_t row = equal_group.template find_best_match<true, false>(30);
+ EXPECT_TRUE(contains_row(row, {1, 2, 3}));
+ EXPECT_EQ((equal_group.template find_best_match<true, true>(30)), 0U);
+
+ row = equal_group.template find_best_match<true, false>(40);
+ EXPECT_TRUE(contains_row(row, {1, 2, 3}));
+
+ row = equal_group.template find_best_match<false, false>(30);
+ EXPECT_TRUE(contains_row(row, {1, 2, 3}));
+
+ EXPECT_EQ((equal_group.template find_best_match<false, true>(30)), 0U);
+
+ row = equal_group.template find_best_match<false, false>(20);
+ EXPECT_TRUE(contains_row(row, {1, 2, 3}));
+}
+
+TEST_F(AsofIndexGroupMatchTest, LargeGroupBinarySearch) {
+ AsofIndexGroup<uint32_t> large_group;
+ for (uint32_t i = 1; i <= 1000; ++i) {
+ large_group.add_row(i * 2, i);
+ }
+ large_group.sort_and_finalize();
+
+ EXPECT_EQ((large_group.template find_best_match<true, false>(1)), 0U);
+ EXPECT_EQ((large_group.template find_best_match<true, false>(500)), 250U);
+ EXPECT_EQ((large_group.template find_best_match<true, false>(999)), 499U);
+ EXPECT_EQ((large_group.template find_best_match<true, false>(1000)), 500U);
+ EXPECT_EQ((large_group.template find_best_match<true, false>(1001)), 500U);
+ EXPECT_EQ((large_group.template find_best_match<true, false>(2000)), 1000U);
+ EXPECT_EQ((large_group.template find_best_match<true, false>(2001)), 1000U);
+
+ EXPECT_EQ((large_group.template find_best_match<false, false>(1)), 1U);
+ EXPECT_EQ((large_group.template find_best_match<false, false>(999)), 500U);
+ EXPECT_EQ((large_group.template find_best_match<false, false>(2001)), 0U);
+
+ EXPECT_EQ((large_group.template find_best_match<true, true>(2)), 0U);
+ EXPECT_EQ((large_group.template find_best_match<true, true>(500)), 249U);
+ EXPECT_EQ((large_group.template find_best_match<true, true>(1000)), 499U);
+ EXPECT_EQ((large_group.template find_best_match<true, true>(2000)), 999U);
+ EXPECT_EQ((large_group.template find_best_match<true, true>(2001)), 1000U);
+
+ EXPECT_EQ((large_group.template find_best_match<false, true>(1)), 1U);
+ EXPECT_EQ((large_group.template find_best_match<false, true>(2)), 2U);
+ EXPECT_EQ((large_group.template find_best_match<false, true>(999)), 500U);
+ EXPECT_EQ((large_group.template find_best_match<false, true>(2000)), 0U);
+ EXPECT_EQ((large_group.template find_best_match<false, true>(2001)), 0U);
+}
+
+TEST_F(AsofIndexGroupMatchTest, EmptyGroupFindBestMatchReturnsZero) {
+ AsofIndexGroup<uint32_t> empty_group;
+ empty_group.sort_and_finalize();
+ EXPECT_EQ((empty_group.template find_best_match<true, false>(42)), 0U);
+ EXPECT_EQ((empty_group.template find_best_match<true, true>(42)), 0U);
+ EXPECT_EQ((empty_group.template find_best_match<false, false>(42)), 0U);
+ EXPECT_EQ((empty_group.template find_best_match<false, true>(42)), 0U);
+}
+
+TEST_F(AsofIndexGroupMatchTest, UInt64FindBestMatch) {
+ auto group64 = make_group<uint64_t>(
+ {{10000000001ULL, 1}, {20000000002ULL, 2}, {30000000003ULL, 3}, {40000000004ULL, 4}});
+ EXPECT_EQ((group64.template find_best_match<true, false>(25000000000ULL)), 2U);
+ EXPECT_EQ((group64.template find_best_match<true, true>(30000000003ULL)), 2U);
+ EXPECT_EQ((group64.template find_best_match<false, false>(25000000000ULL)), 3U);
+ EXPECT_EQ((group64.template find_best_match<false, true>(40000000004ULL)), 0U);
+}
+
+class AsofColumnDispatchTest : public ::testing::Test {};
+
+TEST_F(AsofColumnDispatchTest, DateV2Dispatch) {
+ auto col = make_datev2_column({1, 2, 3});
+ bool dispatched_to_datev2 = false;
+ asof_column_dispatch(col.get(), [&](const auto* typed_col) {
+ if constexpr (std::is_same_v<std::decay_t<decltype(*typed_col)>, ColumnDateV2>) {
+ dispatched_to_datev2 = true;
+ }
+ });
+ EXPECT_TRUE(dispatched_to_datev2);
+}
+
+TEST_F(AsofColumnDispatchTest, DateTimeV2Dispatch) {
+ auto col = ColumnDateTimeV2::create();
+ uint64_t v1 = 1, v2 = 2;
+ col->insert_data(reinterpret_cast<const char*>(&v1), 0);
+ col->insert_data(reinterpret_cast<const char*>(&v2), 0);
+ bool dispatched_to_dtv2 = false;
+ asof_column_dispatch(col.get(), [&](const auto* typed_col) {
+ if constexpr (std::is_same_v<std::decay_t<decltype(*typed_col)>, ColumnDateTimeV2>) {
+ dispatched_to_dtv2 = true;
+ }
+ });
+ EXPECT_TRUE(dispatched_to_dtv2);
+}
+
+TEST_F(AsofColumnDispatchTest, TimestampTZDispatch) {
+ auto col = ColumnTimeStampTz::create();
+ uint64_t v1 = 1, v2 = 2;
+ col->insert_data(reinterpret_cast<const char*>(&v1), 0);
+ col->insert_data(reinterpret_cast<const char*>(&v2), 0);
+ bool dispatched_to_tstz = false;
+ asof_column_dispatch(col.get(), [&](const auto* typed_col) {
+ if constexpr (std::is_same_v<std::decay_t<decltype(*typed_col)>, ColumnTimeStampTz>) {
+ dispatched_to_tstz = true;
+ }
+ });
+ EXPECT_TRUE(dispatched_to_tstz);
+}
+
+TEST_F(AsofColumnDispatchTest, FallbackDispatch) {
+ auto col = make_int32_column({1, 2, 3});
+ bool dispatched_to_icol = false;
+ asof_column_dispatch(col.get(), [&](const auto* typed_col) {
+ if constexpr (std::is_same_v<std::decay_t<decltype(*typed_col)>, IColumn>) {
+ dispatched_to_icol = true;
+ }
+ });
+ EXPECT_TRUE(dispatched_to_icol);
+}
+
+class AsofJoinHelperTest : public ::testing::Test {};
+
+TEST_F(AsofJoinHelperTest, IsAsofJoinRuntime) {
+ EXPECT_TRUE(is_asof_join(TJoinOp::ASOF_LEFT_INNER_JOIN));
+ EXPECT_TRUE(is_asof_join(TJoinOp::ASOF_LEFT_OUTER_JOIN));
+ EXPECT_FALSE(is_asof_join(TJoinOp::INNER_JOIN));
+ EXPECT_FALSE(is_asof_join(TJoinOp::LEFT_OUTER_JOIN));
+}
+
+TEST_F(AsofJoinHelperTest, IsAsofJoinCompileTime) {
+ static_assert(is_asof_join_op_v<TJoinOp::ASOF_LEFT_INNER_JOIN>);
+ static_assert(is_asof_join_op_v<TJoinOp::ASOF_LEFT_OUTER_JOIN>);
+ static_assert(!is_asof_join_op_v<TJoinOp::INNER_JOIN>);
+ static_assert(!is_asof_join_op_v<TJoinOp::LEFT_OUTER_JOIN>);
+
+ static_assert(!is_asof_outer_join_op_v<TJoinOp::ASOF_LEFT_INNER_JOIN>);
+ static_assert(is_asof_outer_join_op_v<TJoinOp::ASOF_LEFT_OUTER_JOIN>);
+ static_assert(!is_asof_outer_join_op_v<TJoinOp::INNER_JOIN>);
+}
+
+class AsofIndexVariantTest : public ::testing::Test {};
+
+TEST_F(AsofIndexVariantTest, MonostateIsDefault) {
+ AsofIndexVariant variant;
+ EXPECT_TRUE(std::holds_alternative<std::monostate>(variant));
+}
+
+TEST_F(AsofIndexVariantTest, EmplaceUint32Groups) {
+ AsofIndexVariant variant;
+ auto& groups = variant.emplace<std::vector<AsofIndexGroup<uint32_t>>>();
+ groups.emplace_back();
+ groups[0].add_row(10, 1);
+ groups[0].sort_and_finalize();
+ EXPECT_TRUE(std::holds_alternative<std::vector<AsofIndexGroup<uint32_t>>>(variant));
+ EXPECT_EQ(std::get<std::vector<AsofIndexGroup<uint32_t>>>(variant)[0].asof_values[0], 10U);
+}
+
+TEST_F(AsofIndexVariantTest, EmplaceUint64Groups) {
+ AsofIndexVariant variant;
+ auto& groups = variant.emplace<std::vector<AsofIndexGroup<uint64_t>>>();
+ groups.emplace_back();
+ groups[0].add_row(99999999999ULL, 1);
+ groups[0].sort_and_finalize();
+ EXPECT_TRUE(std::holds_alternative<std::vector<AsofIndexGroup<uint64_t>>>(variant));
+ EXPECT_EQ(std::get<std::vector<AsofIndexGroup<uint64_t>>>(variant)[0].asof_values[0],
+ 99999999999ULL);
+}
+
+} // namespace doris
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 2b6c63377cf..1175712cc24 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -83,6 +83,7 @@ APPEND: 'APPEND';
ARRAY: 'ARRAY';
AS: 'AS';
ASC: 'ASC';
+ASOF: 'ASOF';
AT: 'AT';
AUTHORS: 'AUTHORS';
AUTO: 'AUTO';
@@ -355,6 +356,7 @@ MATCH: 'MATCH';
MATCHED: 'MATCHED';
MATCH_ALL: 'MATCH_ALL';
MATCH_ANY: 'MATCH_ANY';
+MATCH_CONDITION: 'MATCH_CONDITION';
MATCH_NAME: 'MATCH_NAME';
MATCH_NAME_GLOB: 'MATCH_NAME_GLOB';
MATCH_PHRASE: 'MATCH_PHRASE';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index d5999334468..e07b45f751f 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -1302,7 +1302,11 @@ relation
;
joinRelation
- : (joinType) JOIN distributeType? right=relationPrimary joinCriteria?
+ : (joinType) JOIN distributeType? right=relationPrimary matchCondition? joinCriteria?
+ ;
+
+matchCondition
+ : MATCH_CONDITION LEFT_PAREN valueExpression RIGHT_PAREN
;
// Just like `opt_plan_hints` in legacy CUP parser.
@@ -1413,6 +1417,8 @@ joinType
| RIGHT SEMI
| LEFT ANTI
| RIGHT ANTI
+ | ASOF LEFT?
+ | ASOF INNER
;
joinCriteria
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java
index bd8798bd4d4..54f85e765db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JoinOperator.java
@@ -39,7 +39,11 @@ public enum JoinOperator {
NULL_AWARE_LEFT_ANTI_JOIN("NULL AWARE LEFT ANTI JOIN",
TJoinOp.NULL_AWARE_LEFT_ANTI_JOIN),
NULL_AWARE_LEFT_SEMI_JOIN("NULL AWARE LEFT SEMI JOIN",
- TJoinOp.NULL_AWARE_LEFT_SEMI_JOIN);
+ TJoinOp.NULL_AWARE_LEFT_SEMI_JOIN),
+ ASOF_LEFT_INNER_JOIN("ASOF_LEFT_INNER_JOIN", TJoinOp.ASOF_LEFT_INNER_JOIN),
+ ASOF_RIGHT_INNER_JOIN("ASOF_RIGHT_INNER_JOIN", TJoinOp.ASOF_RIGHT_INNER_JOIN),
+ ASOF_LEFT_OUTER_JOIN("ASOF_LEFT_OUTER_JOIN", TJoinOp.ASOF_LEFT_OUTER_JOIN),
+ ASOF_RIGHT_OUTER_JOIN("ASOF_RIGHT_OUTER_JOIN", TJoinOp.ASOF_RIGHT_OUTER_JOIN);
private final String description;
private final TJoinOp thriftJoinOp;
@@ -57,66 +61,4 @@ public enum JoinOperator {
public TJoinOp toThrift() {
return thriftJoinOp;
}
-
- public boolean isOuterJoin() {
- return this == LEFT_OUTER_JOIN || this == RIGHT_OUTER_JOIN || this == FULL_OUTER_JOIN;
- }
-
- public boolean isSemiAntiJoin() {
- return this == LEFT_SEMI_JOIN || this == RIGHT_SEMI_JOIN || this == LEFT_ANTI_JOIN
- || this == NULL_AWARE_LEFT_ANTI_JOIN || this == RIGHT_ANTI_JOIN;
- }
-
- public boolean isSemiJoin() {
- return this == JoinOperator.LEFT_SEMI_JOIN || this == JoinOperator.LEFT_ANTI_JOIN
- || this == JoinOperator.RIGHT_SEMI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN
- || this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
- }
-
- public boolean isSemiOrAntiJoinNoNullAware() {
- return this == JoinOperator.LEFT_SEMI_JOIN || this == JoinOperator.LEFT_ANTI_JOIN
- || this == JoinOperator.RIGHT_SEMI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN;
- }
-
- public boolean isAntiJoinNullAware() {
- return this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
- }
-
- public boolean isAntiJoinNoNullAware() {
- return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN;
- }
-
- public boolean isLeftSemiJoin() {
- return this.thriftJoinOp == TJoinOp.LEFT_SEMI_JOIN;
- }
-
- public boolean isInnerJoin() {
- return this.thriftJoinOp == TJoinOp.INNER_JOIN;
- }
-
- public boolean isAntiJoin() {
- return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.RIGHT_ANTI_JOIN
- || this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
- }
-
- public boolean supportMarkJoin() {
- return this == JoinOperator.LEFT_ANTI_JOIN || this == JoinOperator.LEFT_SEMI_JOIN
- || this == JoinOperator.CROSS_JOIN || this == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
- }
-
- public boolean isCrossJoin() {
- return this == CROSS_JOIN;
- }
-
- public boolean isFullOuterJoin() {
- return this == FULL_OUTER_JOIN;
- }
-
- public boolean isLeftOuterJoin() {
- return this == LEFT_OUTER_JOIN;
- }
-
- public boolean isRightOuterJoin() {
- return this == RIGHT_OUTER_JOIN;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 7f22f4aa59c..cf40d270365 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -1631,10 +1631,16 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
joinOperator = JoinOperator.NULL_AWARE_LEFT_SEMI_JOIN;
}
}
+ Expr matchCondition = null;
+ if (joinType.isAsofJoin()) {
+ Preconditions.checkState(hashJoin.getOtherJoinConjuncts().size() == 1,
+ String.format("asof join's match condition is invalid %s", hashJoin.getOtherJoinConjuncts()));
+ matchCondition = ExpressionTranslator.translate(hashJoin.getOtherJoinConjuncts().get(0), context);
+ }
HashJoinNode hashJoinNode = new HashJoinNode(context.nextPlanNodeId(), leftPlanRoot,
- rightPlanRoot, joinOperator, execEqConjuncts, Lists.newArrayList(), markConjuncts,
- null, null, null, hashJoin.isMarkJoin());
+ rightPlanRoot, joinOperator, execEqConjuncts, Lists.newArrayList(), matchCondition,
+ markConjuncts, null, null, null, hashJoin.isMarkJoin());
hashJoinNode.setNereidsId(hashJoin.getId());
context.getNereidsIdToPlanNodeIdMap().put(hashJoin.getId(), hashJoinNode.getId());
hashJoinNode.setChildrenDistributeExprLists(distributeExprLists);
@@ -1802,16 +1808,30 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
// set slots as nullable for outer join
- if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) {
- rightIntermediateSlotDescriptor.forEach(sd -> sd.setIsNullable(true));
+ if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.ASOF_LEFT_OUTER_JOIN
+ || joinType == JoinType.FULL_OUTER_JOIN) {
+ for (SlotDescriptor sd : rightIntermediateSlotDescriptor) {
+ sd.setIsNullable(true);
+ SlotRef slotRef = new SlotRef(sd);
+ ExprId exprId = context.findExprId(sd.getId());
+ context.addExprIdSlotRefPair(exprId, slotRef);
+ }
}
- if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) {
- leftIntermediateSlotDescriptor.forEach(sd -> sd.setIsNullable(true));
+ if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.ASOF_RIGHT_OUTER_JOIN
+ || joinType == JoinType.FULL_OUTER_JOIN) {
+ for (SlotDescriptor sd : leftIntermediateSlotDescriptor) {
+ sd.setIsNullable(true);
+ SlotRef slotRef = new SlotRef(sd);
+ ExprId exprId = context.findExprId(sd.getId());
+ context.addExprIdSlotRefPair(exprId, slotRef);
+ }
}
// Constant expr will cause be crash.
// But EliminateJoinCondition and Expression Rewrite already eliminate true literal.
- List<Expr> otherJoinConjuncts = hashJoin.getOtherJoinConjuncts()
+ // asof's other conjunct is actually match_condition, and be translated as hash conjuncts already
+ List<Expr> otherJoinConjuncts = joinType.isAsofJoin() ? ImmutableList.of() :
+ hashJoin.getOtherJoinConjuncts()
.stream()
.map(e -> ExpressionTranslator.translate(e, context))
.collect(Collectors.toList());
@@ -1977,11 +1997,23 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
}
// set slots as nullable for outer join
- if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) {
- rightIntermediateSlotDescriptor.forEach(sd -> sd.setIsNullable(true));
- }
- if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) {
- leftIntermediateSlotDescriptor.forEach(sd -> sd.setIsNullable(true));
+ if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.ASOF_LEFT_OUTER_JOIN
+ || joinType == JoinType.FULL_OUTER_JOIN) {
+ for (SlotDescriptor sd : rightIntermediateSlotDescriptor) {
+ sd.setIsNullable(true);
+ SlotRef slotRef = new SlotRef(sd);
+ ExprId exprId = context.findExprId(sd.getId());
+ context.addExprIdSlotRefPair(exprId, slotRef);
+ }
+ }
+ if (joinType == JoinType.RIGHT_OUTER_JOIN || joinType == JoinType.ASOF_RIGHT_OUTER_JOIN
+ || joinType == JoinType.FULL_OUTER_JOIN) {
+ for (SlotDescriptor sd : leftIntermediateSlotDescriptor) {
+ sd.setIsNullable(true);
+ SlotRef slotRef = new SlotRef(sd);
+ ExprId exprId = context.findExprId(sd.getId());
+ context.addExprIdSlotRefPair(exprId, slotRef);
+ }
}
nestedLoopJoinNode.setvIntermediateTupleDescList(Lists.newArrayList(intermediateDescriptor));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index a2b5053785a..66867b9272e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -529,6 +529,7 @@ import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.DefaultValueSlot;
import org.apache.doris.nereids.trees.expressions.DereferenceExpression;
import org.apache.doris.nereids.trees.expressions.Divide;
+import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Exists;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -4299,6 +4300,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) {
LogicalPlan last = input;
for (JoinRelationContext join : ctx.joinRelation()) {
+ boolean isAsofJoin = false;
JoinType joinType;
if (join.joinType().CROSS() != null) {
joinType = JoinType.CROSS_JOIN;
@@ -4316,6 +4318,10 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
} else {
joinType = JoinType.RIGHT_ANTI_JOIN;
}
+ } else if (join.joinType().ASOF() != null) {
+ joinType = join.joinType().INNER() != null
+ ? JoinType.ASOF_LEFT_INNER_JOIN : JoinType.ASOF_LEFT_OUTER_JOIN;
+ isAsofJoin = true;
} else if (join.joinType().LEFT() != null) {
joinType = JoinType.LEFT_OUTER_JOIN;
} else if (join.joinType().RIGHT() != null) {
@@ -4327,6 +4333,23 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
} else {
joinType = JoinType.CROSS_JOIN;
}
+ Expression matchCondition = null;
+ if (join.matchCondition() != null) {
+ if (!isAsofJoin) {
+ throw new ParseException("only ASOF JOIN support MATCH_CONDITION", join);
+ }
+ matchCondition = typedVisit(join.matchCondition().valueExpression());
+ if (!(matchCondition instanceof LessThan
+ || matchCondition instanceof LessThanEqual
+ || matchCondition instanceof GreaterThan
+ || matchCondition instanceof GreaterThanEqual)) {
+ throw new ParseException("ASOF JOIN's MATCH_CONDITION must be <, <=, >, >=", join);
+ }
+ } else {
+ if (isAsofJoin) {
+ throw new ParseException("ASOF JOIN must specify MATCH_CONDITION", join);
+ }
+ }
DistributeHint distributeHint = new DistributeHint(DistributeType.NONE);
if (join.distributeType() != null) {
distributeHint = visitDistributeType(join.distributeType());
@@ -4347,22 +4370,45 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
.collect(ImmutableList.toImmutableList());
}
} else {
+ if (isAsofJoin) {
+ throw new ParseException("ASOF JOIN must have on or using clause", join);
+ }
// keep same with original planner, allow cross/inner join
if (!joinType.isInnerOrCrossJoin()) {
throw new ParseException("on mustn't be empty except for cross/inner join", join);
}
}
+
if (ids == null) {
- last = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION,
- condition.map(ExpressionUtils::extractConjunction)
- .orElse(ExpressionUtils.EMPTY_CONDITION),
- distributeHint,
- Optional.empty(),
- last,
- plan(join.relationPrimary()), null);
+ if (isAsofJoin) {
+ if (!condition.isPresent()) {
+ throw new ParseException("ASOF JOIN can't be used without ON clause", join);
+ }
+ List<Expression> conjuncts = ExpressionUtils.extractConjunction(condition.get());
+ for (Expression expression : conjuncts) {
+ if (!(expression instanceof EqualPredicate)) {
+ throw new ParseException("ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts",
+ join);
+ }
+ }
+ last = new LogicalJoin<>(joinType, conjuncts,
+ ImmutableList.of(matchCondition),
+ distributeHint,
+ Optional.empty(),
+ last,
+ plan(join.relationPrimary()), null);
+ } else {
+ last = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION,
+ condition.map(ExpressionUtils::extractConjunction)
+ .orElse(ExpressionUtils.EMPTY_CONDITION),
+ distributeHint,
+ Optional.empty(),
+ last,
+ plan(join.relationPrimary()), null);
+ }
} else {
- last = new LogicalUsingJoin<>(joinType, last, plan(join.relationPrimary()), ids, distributeHint);
-
+ last = new LogicalUsingJoin<>(joinType, last, plan(join.relationPrimary()), ids,
+ matchCondition != null ? Optional.of(matchCondition) : Optional.empty(), distributeHint);
}
if (distributeHint.distributeType != DistributeType.NONE
&& ConnectContext.get().getStatementContext() != null
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
index f3dd019c462..d032fc6eea1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
@@ -86,6 +86,7 @@ public class RuntimeFilterGenerator extends PlanPostProcessor {
JoinType.LEFT_ANTI_JOIN,
JoinType.FULL_OUTER_JOIN,
JoinType.LEFT_OUTER_JOIN,
+ JoinType.ASOF_LEFT_OUTER_JOIN,
JoinType.NULL_AWARE_LEFT_ANTI_JOIN
);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
index adbb5e2c863..7c7f94db657 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java
@@ -245,6 +245,9 @@ public class RuntimeFilterPushDownVisitor extends PlanVisitor<Boolean, PushDownC
&& !join.getOutputSet().containsAll(ctx.probeExpr.getInputSlots())) {
return false;
}
+ if (join.getJoinType().isAsofJoin()) {
+ return false;
+ }
boolean pushed = false;
if (ctx.builderNode instanceof PhysicalHashJoin) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java
index bf21d5baf9e..789ab2c0759 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopnFilterPushDownVisitor.java
@@ -195,7 +195,7 @@ public class TopnFilterPushDownVisitor extends PlanVisitor<Boolean, PushDownCont
@Override
public Boolean visitPhysicalHashJoin(PhysicalHashJoin<? extends Plan, ? extends Plan> join,
PushDownContext ctx) {
- if (ctx.nullsFirst && join.getJoinType().isOuterJoin()) {
+ if (ctx.nullsFirst && (join.getJoinType().isOuterJoin() || join.getJoinType().isAsofOuterJoin())) {
// topn-filter can be pushed down to the left child of leftOuterJoin
// and to the right child of rightOuterJoin,
// but PushDownTopNThroughJoin rule already pushes topn to the left and right side.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java
index 6c22917bb23..bc54aa84bd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java
@@ -238,9 +238,11 @@ public class LazySlotPruning extends DefaultPlanRewriter<LazySlotPruning.Context
SlotReference rowIdSlot = (SlotReference) plan.getOutput().get(rowIdPos);
if (join.getJoinType().isFullOuterJoin()) {
context.updateRowIdSlot(rowIdSlot.withNullable(true));
- } else if (join.getJoinType().isLeftOuterJoin() && plan.child(1).getOutput().contains(rowIdSlot)) {
+ } else if ((join.getJoinType().isLeftOuterJoin() || join.getJoinType().isAsofLeftOuterJoin())
+ && plan.child(1).getOutput().contains(rowIdSlot)) {
context.updateRowIdSlot(rowIdSlot.withNullable(true));
- } else if (join.getJoinType().isRightOuterJoin() && plan.child(0).getOutput().contains(rowIdSlot)) {
+ } else if ((join.getJoinType().isRightOuterJoin() || join.getJoinType().isAsofRightOuterJoin())
+ && plan.child(0).getOutput().contains(rowIdSlot)) {
context.updateRowIdSlot(rowIdSlot.withNullable(true));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/PushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/PushDownVisitor.java
index 05fceadad54..2dbc6598718 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/PushDownVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/runtimefilterv2/PushDownVisitor.java
@@ -132,7 +132,7 @@ public class PushDownVisitor extends PlanVisitor<Boolean, PushDownContext> {
return false;
}
- if (ctx.isNullSafe() || join.getJoinType().isOuterJoin()) {
+ if (ctx.isNullSafe() || join.getJoinType().isOuterJoin() || join.getJoinType().isAsofJoin()) {
return false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index c553f9f8be5..2a1a54435a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -525,6 +525,8 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
switch (hashJoin.getJoinType()) {
case INNER_JOIN:
case CROSS_JOIN:
+ case ASOF_LEFT_INNER_JOIN:
+ case ASOF_RIGHT_INNER_JOIN:
if (shuffleSide == ShuffleSide.LEFT) {
return new PhysicalProperties(
DistributionSpecHash.merge(rightHashSpec, leftHashSpec, outputShuffleType)
@@ -545,6 +547,7 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
case LEFT_ANTI_JOIN:
case NULL_AWARE_LEFT_ANTI_JOIN:
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
if (shuffleSide == ShuffleSide.LEFT || shuffleSide == ShuffleSide.BOTH) {
return new PhysicalProperties(
leftHashSpec.withShuffleTypeAndForbidColocateJoin(outputShuffleType)
@@ -557,6 +560,7 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
case RIGHT_SEMI_JOIN:
case RIGHT_ANTI_JOIN:
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
if (shuffleSide == ShuffleSide.RIGHT || shuffleSide == ShuffleSide.BOTH) {
return new PhysicalProperties(
rightHashSpec.withShuffleTypeAndForbidColocateJoin(outputShuffleType)
@@ -593,6 +597,8 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
DistributionSpecHash leftHashSpec, DistributionSpecHash rightHashSpec) {
switch (hashJoin.getJoinType()) {
case INNER_JOIN:
+ case ASOF_LEFT_INNER_JOIN:
+ case ASOF_RIGHT_INNER_JOIN:
case CROSS_JOIN:
return new PhysicalProperties(DistributionSpecHash.merge(
leftHashSpec, rightHashSpec, leftHashSpec.getShuffleType()));
@@ -600,10 +606,12 @@ public class ChildOutputPropertyDeriver extends PlanVisitor<PhysicalProperties,
case LEFT_ANTI_JOIN:
case NULL_AWARE_LEFT_ANTI_JOIN:
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
return new PhysicalProperties(leftHashSpec);
case RIGHT_SEMI_JOIN:
case RIGHT_ANTI_JOIN:
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec, hashJoin.getHashJoinConjuncts())) {
return new PhysicalProperties(rightHashSpec);
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
index af4baee6043..aa556536aba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
@@ -344,6 +344,7 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<List<List<PhysicalP
DistributionSpecHash rightHashSpec) {
boolean isJoinTypeInScope = (joinType == JoinType.RIGHT_ANTI_JOIN
|| joinType == JoinType.RIGHT_OUTER_JOIN
+ || joinType == JoinType.ASOF_RIGHT_OUTER_JOIN
|| joinType == JoinType.FULL_OUTER_JOIN);
boolean isSpecInScope = (leftHashSpec.getShuffleType() == ShuffleType.NATURAL
|| rightHashSpec.getShuffleType() == ShuffleType.NATURAL);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
index 9409261fc52..af06b10c639 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
@@ -41,6 +41,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.BoundStar;
+import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.DefaultValueSlot;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.ExprId;
@@ -96,12 +97,14 @@ import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.JoinUtils;
import org.apache.doris.nereids.util.PlanUtils;
import org.apache.doris.nereids.util.PlanUtils.CollectNonWindowedAggFuncs;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.SqlModeHelper;
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
@@ -614,8 +617,51 @@ public class BindExpression implements AnalysisRuleFactory {
otherJoinConjuncts.add(conjunct);
}
}
+ List<Expression> hashConjuncts = hashJoinConjuncts.build();
+ List<Expression> otherConjuncts = otherJoinConjuncts.build();
+ if (join.getJoinType().isAsofJoin()) {
+ // validate hash conjuncts
+ if (!JoinUtils.extractExpressionForHashTable(join.left().getOutput(), join.right().getOutput(),
+ hashConjuncts).second.isEmpty()) {
+ throw new AnalysisException(String.format("ASOF join's hash conjuncts must be"
+ + " in form of t1.col = t2.col but its %s", hashConjuncts));
+ }
+ // validate match_condition's data type and normalize expression order
+ // After normalization: left side of MATCH_CONDITION references left table (probe),
+ // right side references right table (build). This is required by BE.
+ boolean isValid = false;
+ if (otherConjuncts.size() == 1) {
+ Expression conjunct = otherConjuncts.get(0);
+ if (conjunct instanceof ComparisonPredicate) {
+ if (conjunct.child(0).getDataType().isDateLikeType()
+ && conjunct.child(1).getDataType().isDateLikeType()) {
+ Set<Slot> leftSlots = join.left().getOutputSet();
+ Set<Slot> rightSlots = join.right().getOutputSet();
+ Set<Slot> leftUsedSlots = conjunct.child(0).getInputSlots();
+ Set<Slot> rightUsedSlots = conjunct.child(1).getInputSlots();
+ if (rightSlots.containsAll(leftUsedSlots) && leftSlots.containsAll(rightUsedSlots)) {
+ Set<Slot> temp = leftSlots;
+ leftSlots = rightSlots;
+ rightSlots = temp;
+ otherConjuncts = ImmutableList.of(((ComparisonPredicate) conjunct).commute());
+ }
+ if (!(leftSlots.containsAll(leftUsedSlots) && rightSlots.containsAll(rightUsedSlots))) {
+ throw new AnalysisException("MATCH_CONDITION must specify column from both table");
+ }
+ isValid = true;
+ } else {
+ throw new AnalysisException("only allow date, datetime and timestamptz in MATCH_CONDITION");
+ }
+ }
+ }
+ if (!isValid) {
+ // other unexpected error
+ throw new AnalysisException(String.format("MATCH_CONDITION is invalid %s",
+ Joiner.on(",").join(otherConjuncts)));
+ }
+ }
return new LogicalJoin<>(join.getJoinType(),
- hashJoinConjuncts.build(), otherJoinConjuncts.build(),
+ hashConjuncts, otherConjuncts,
join.getDistributeHint(), join.getMarkJoinSlotReference(), join.getExceptAsteriskOutputs(),
join.children(), null);
}
@@ -685,10 +731,10 @@ public class BindExpression implements AnalysisRuleFactory {
}
return new LogicalJoin<>(
- using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(),
- hashEqExprs.build(), ImmutableList.of(),
- using.getDistributeHint(), Optional.empty(), rightConjunctsSlots,
- using.children(), null);
+ using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(),
+ hashEqExprs.build(), using.getMatchCondition().map(ImmutableList::of).orElse(ImmutableList.of()),
+ using.getDistributeHint(), Optional.empty(), rightConjunctsSlots,
+ using.children(), null);
}
private Plan bindProject(MatchingContext<LogicalProject<Plan>> ctx) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java
index 4a40bed1e50..b8122f5f8e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java
@@ -65,6 +65,7 @@ public class JoinCommute extends OneExplorationRuleFactory {
// null aware mark join will be translated to null aware left semi/anti join
// we don't support null aware right semi/anti join, so should not commute
.whenNot(join -> JoinUtils.isNullAwareMarkJoin(join))
+ .whenNot(join -> join.getJoinType().isAsofJoin())
// commuting nest loop mark join or left anti mark join is not supported by be
.whenNot(join -> join.isMarkJoin() && (join.getHashJoinConjuncts().isEmpty()
|| join.getJoinType().isLeftAntiJoin()))
@@ -74,8 +75,8 @@ public class JoinCommute extends OneExplorationRuleFactory {
&& NestedLoopJoinNode.canParallelize(JoinType.toJoinOperator(join.getJoinType()))
&& !NestedLoopJoinNode.canParallelize(JoinType.toJoinOperator(join.getJoinType().swap())))
.then(join -> {
- LogicalJoin<? extends Plan, ? extends Plan> newJoin = join.withTypeChildren(
- join.getJoinType().swap(), join.right(), join.left(), null);
+ // Use swap() to properly handle ASOF JOIN MATCH_CONDITION commutation
+ LogicalJoin<? extends Plan, ? extends Plan> newJoin = join.swap();
newJoin.getJoinReorderContext().copyFrom(join.getJoinReorderContext());
newJoin.getJoinReorderContext().setHasCommute(true);
if (swapType == SwapType.ZIG_ZAG && isNotBottomJoin(join)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java
index b63405eba62..905af662a2d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoin.java
@@ -58,7 +58,9 @@ public class PushDownProjectThroughInnerOuterJoin implements ExplorationRuleFact
return ImmutableList.of(
logicalJoin(logicalProject(logicalJoin()), group())
.when(j -> j.left().child().getJoinType().isOuterJoin()
- || j.left().child().getJoinType().isInnerJoin())
+ || j.left().child().getJoinType().isInnerJoin()
+ || j.left().child().getJoinType().isAsofOuterJoin()
+ || j.left().child().getJoinType().isAsofInnerJoin())
// Just pushdown project with non-column expr like (t.id + 1)
.whenNot(j -> j.left().isAllSlots())
.whenNot(j -> j.left().child().hasDistributeHint())
@@ -72,7 +74,9 @@ public class PushDownProjectThroughInnerOuterJoin implements ExplorationRuleFact
}).toRule(RuleType.PUSH_DOWN_PROJECT_THROUGH_INNER_OUTER_JOIN_LEFT),
logicalJoin(group(), logicalProject(logicalJoin()))
.when(j -> j.right().child().getJoinType().isOuterJoin()
- || j.right().child().getJoinType().isInnerJoin())
+ || j.right().child().getJoinType().isInnerJoin()
+ || j.right().child().getJoinType().isAsofOuterJoin()
+ || j.right().child().getJoinType().isAsofInnerJoin())
// Just pushdown project with non-column expr like (t.id + 1)
.whenNot(j -> j.right().isAllSlots())
.whenNot(j -> j.right().child().hasDistributeHint())
@@ -104,7 +108,7 @@ public class PushDownProjectThroughInnerOuterJoin implements ExplorationRuleFact
List<NamedExpression> aProjects = new ArrayList<>();
List<NamedExpression> bProjects = new ArrayList<>();
List<NamedExpression> projects;
- if (join.getJoinType().isInnerJoin()) {
+ if (join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()) {
projects = project.getProjects();
} else {
Map<Slot, Slot> childrenSlots = new HashMap<>();
@@ -136,8 +140,10 @@ public class PushDownProjectThroughInnerOuterJoin implements ExplorationRuleFact
return null;
}
// we could not push nullable side project
- if (((join.getJoinType().isLeftOuterJoin() || join.getJoinType().isFullOuterJoin()) && rightContains)
- || ((join.getJoinType().isRightOuterJoin() || join.getJoinType().isFullOuterJoin()) && leftContains)) {
+ if (((join.getJoinType().isLeftOuterJoin() || join.getJoinType().isAsofLeftOuterJoin()
+ || join.getJoinType().isFullOuterJoin()) && rightContains)
+ || ((join.getJoinType().isRightOuterJoin() || join.getJoinType().isAsofRightOuterJoin()
+ || join.getJoinType().isFullOuterJoin()) && leftContains)) {
return null;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index c4b18a96b84..9f76c242dcf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -104,7 +104,11 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
JoinType.RIGHT_SEMI_JOIN,
JoinType.LEFT_ANTI_JOIN,
JoinType.RIGHT_ANTI_JOIN,
- JoinType.NULL_AWARE_LEFT_ANTI_JOIN);
+ JoinType.NULL_AWARE_LEFT_ANTI_JOIN,
+ JoinType.ASOF_LEFT_INNER_JOIN,
+ JoinType.ASOF_RIGHT_INNER_JOIN,
+ JoinType.ASOF_LEFT_OUTER_JOIN,
+ JoinType.ASOF_RIGHT_OUTER_JOIN);
/**
* The abstract template method for query rewrite, it contains the main logic, try to rewrite query by
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index 8a6ed611cd1..11f2e7c409d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -64,6 +64,7 @@ import javax.annotation.Nullable;
public class HyperGraphComparator {
// This second join can be inferred to the first join by map value,
// The map value means the child's should be no-nullable
+ // please aware ASOF OUTER JOIN can only be converted to ASOF INNER JOIN
static Map<Pair<JoinType, JoinType>, Pair<Boolean, Boolean>> canInferredJoinTypeMap = ImmutableMap
.<Pair<JoinType, JoinType>, Pair<Boolean, Boolean>>builder()
.put(Pair.of(JoinType.LEFT_SEMI_JOIN, JoinType.INNER_JOIN), Pair.of(false, false))
@@ -71,6 +72,8 @@ public class HyperGraphComparator {
.put(Pair.of(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN), Pair.of(false, true))
.put(Pair.of(JoinType.INNER_JOIN, JoinType.RIGHT_OUTER_JOIN), Pair.of(true, false))
.put(Pair.of(JoinType.INNER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, true))
+ .put(Pair.of(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN), Pair.of(false, true))
+ .put(Pair.of(JoinType.ASOF_RIGHT_INNER_JOIN, JoinType.ASOF_RIGHT_OUTER_JOIN), Pair.of(true, false))
.put(Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, false))
.put(Pair.of(JoinType.RIGHT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(false, true))
.build();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java
index 72283f46a49..3f718a2862c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java
@@ -233,6 +233,9 @@ public class ConstantPropagation extends DefaultPlanRewriter<ExpressionRewriteCo
// update: BE not support nested loop mark join, so mark join condition need to be an equation,
// but constant propagation may rewrite an equal to TRUE/FALSE/NULL, so we don't rewrite mark join condition.
join = visitChildren(this, join, context);
+ if (join.getJoinType().isAsofJoin()) {
+ return join;
+ }
List<Expression> newHashJoinConjuncts = join.getHashJoinConjuncts();
List<Expression> newOtherJoinConjuncts = join.getOtherJoinConjuncts();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java
index 46445573055..8791bb693f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java
@@ -79,7 +79,8 @@ public class ConvertOuterJoinToAntiJoin extends DefaultPlanRewriter<Map<ExprId,
private Plan toAntiJoin(LogicalFilter<LogicalJoin<Plan, Plan>> filter, Map<ExprId, ExprId> replaceMap) {
LogicalJoin<Plan, Plan> join = filter.child();
- if (!join.getJoinType().isLeftOuterJoin() && !join.getJoinType().isRightOuterJoin()) {
+ if (join.getJoinType().isAsofJoin()
+ || (!join.getJoinType().isLeftOuterJoin() && !join.getJoinType().isRightOuterJoin())) {
return filter;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java
index 55e79e4742e..354cea82b16 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateEmptyRelation.java
@@ -231,12 +231,14 @@ public class EliminateEmptyRelation implements RewriteRuleFactory {
}
private boolean canReplaceJoinByEmptyRelation(LogicalJoin<?, ?> join) {
- return !join.isMarkJoin() && ((join.getJoinType() == JoinType.INNER_JOIN
+ return !join.isMarkJoin() && ((join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()
|| join.getJoinType() == JoinType.LEFT_SEMI_JOIN
|| join.getJoinType() == JoinType.RIGHT_SEMI_JOIN
|| join.getJoinType() == JoinType.CROSS_JOIN)
- || (join.getJoinType() == JoinType.LEFT_OUTER_JOIN && join.left() instanceof EmptyRelation)
- || (join.getJoinType() == JoinType.RIGHT_OUTER_JOIN && join.right() instanceof EmptyRelation));
+ || (join.getJoinType().isLeftOuterJoin() && join.left() instanceof EmptyRelation)
+ || (join.getJoinType().isAsofLeftOuterJoin() && join.left() instanceof EmptyRelation)
+ || (join.getJoinType().isRightOuterJoin() && join.right() instanceof EmptyRelation)
+ || (join.getJoinType().isAsofRightOuterJoin() && join.right() instanceof EmptyRelation));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
index b8704f5610b..435d280ccf6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
@@ -49,7 +49,7 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(
- logicalJoin().when(join -> join.getJoinType().isOuterJoin())
+ logicalJoin().when(join -> join.getJoinType().isOuterJoin() || join.getJoinType().isAsofOuterJoin())
).then(filter -> {
LogicalJoin<Plan, Plan> join = filter.child();
@@ -79,7 +79,7 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory {
conjunctsChanged |= conjuncts.add(isNotNull);
}
}
- if (newJoinType.isInnerJoin()) {
+ if (newJoinType.isInnerJoin() || newJoinType.isAsofInnerJoin()) {
/*
* for example: (A left join B on A.a=B.b) join C on B.x=C.x
* inner join condition B.x=C.x implies 'B.x is not null',
@@ -116,9 +116,15 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory {
if (joinType.isRightOuterJoin() && canFilterLeftNull) {
return JoinType.INNER_JOIN;
}
+ if (joinType.isAsofRightOuterJoin() && canFilterLeftNull) {
+ return JoinType.ASOF_RIGHT_INNER_JOIN;
+ }
if (joinType.isLeftOuterJoin() && canFilterRightNull) {
return JoinType.INNER_JOIN;
}
+ if (joinType.isAsofLeftOuterJoin() && canFilterRightNull) {
+ return JoinType.ASOF_LEFT_INNER_JOIN;
+ }
if (joinType.isFullOuterJoin() && canFilterLeftNull && canFilterRightNull) {
return JoinType.INNER_JOIN;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferJoinNotNull.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferJoinNotNull.java
index 39a0e63ff21..5f87a3ec940 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferJoinNotNull.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferJoinNotNull.java
@@ -42,7 +42,8 @@ public class InferJoinNotNull extends OneRewriteRuleFactory {
public Rule build() {
// TODO: maybe consider ANTI?
return logicalJoin(any(), any())
- .when(join -> join.getJoinType().isInnerJoin() || join.getJoinType().isSemiJoin())
+ .when(join -> join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()
+ || join.getJoinType().isSemiJoin())
.whenNot(LogicalJoin::isMarkJoin)
.thenApply(ctx -> {
LogicalJoin<Plan, Plan> join = ctx.root;
@@ -52,7 +53,7 @@ public class InferJoinNotNull extends OneRewriteRuleFactory {
Plan left = join.left();
Plan right = join.right();
- if (join.getJoinType().isInnerJoin()) {
+ if (join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()) {
Set<Expression> leftNotNull = ExpressionUtils.inferNotNull(
conjuncts, join.left().getOutputSet(), ctx.cascadesContext);
Set<Expression> rightNotNull = ExpressionUtils.inferNotNull(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java
index f0d4deac147..370af833f98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java
@@ -97,23 +97,32 @@ public class InferPredicates extends DefaultPlanRewriter<JobContext> implements
}
Plan left = join.left();
Plan right = join.right();
- Set<Expression> expressions = getAllExpressions(left, right, join.getOnClauseCondition());
+ Set<Expression> expressions;
+ if (join.getJoinType().isAsofJoin()) {
+ expressions = getAllExpressions(left, right, ExpressionUtils.optionalAnd(join.getHashJoinConjuncts()));
+ } else {
+ expressions = getAllExpressions(left, right, join.getOnClauseCondition());
+ }
switch (join.getJoinType()) {
case CROSS_JOIN:
left = inferNewPredicate(left, expressions);
right = inferNewPredicate(right, expressions);
break;
case INNER_JOIN:
+ case ASOF_LEFT_INNER_JOIN:
+ case ASOF_RIGHT_INNER_JOIN:
case LEFT_SEMI_JOIN:
case RIGHT_SEMI_JOIN:
left = inferNewPredicateRemoveUselessIsNull(left, expressions, join, context.getCascadesContext());
right = inferNewPredicateRemoveUselessIsNull(right, expressions, join, context.getCascadesContext());
break;
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
case LEFT_ANTI_JOIN:
right = inferNewPredicateRemoveUselessIsNull(right, expressions, join, context.getCascadesContext());
break;
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
case RIGHT_ANTI_JOIN:
left = inferNewPredicateRemoveUselessIsNull(left, expressions, join, context.getCascadesContext());
break;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InitJoinOrder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InitJoinOrder.java
index 5129f3fdfd8..3711f4b9c91 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InitJoinOrder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InitJoinOrder.java
@@ -62,10 +62,11 @@ public class InitJoinOrder extends OneRewriteRuleFactory {
}
private Plan swapJoinChildrenIfNeed(LogicalJoin<? extends Plan, ? extends Plan> join, CascadesContext context) {
- if (join.getJoinType().isLeftSemiOrAntiJoin()) {
- // TODO: currently, the transform rules for right semi/anti join is not complete,
+ if (join.getJoinType().isLeftSemiOrAntiJoin() || join.getJoinType().isAsofJoin()) {
+ // TODO: currently, the transform rules for right semi/anti/asof join is not complete,
// for example LogicalJoinSemiJoinTransposeProject (tpch 22) only works for left semi/anti join
// if we swap left semi/anti to right semi/anti, we lost the opportunity to optimize join order
+ // and for asof join, the asof right join's performance is poor, we also disable swap
return null;
}
List<CatalogRelation> scans = join.collectToList(CatalogRelation.class::isInstance);
@@ -87,9 +88,8 @@ public class InitJoinOrder extends OneRewriteRuleFactory {
// requires "left.getStats().getRowCount() > 0" to avoid dead loop when negative row count is estimated.
if (left.getStats().getRowCount() < right.getStats().getRowCount() * SWAP_THRESHOLD
&& left.getStats().getRowCount() > 0) {
- join = join.withTypeChildren(swapType, right, left,
- join.getJoinReorderContext());
- return join;
+ // Use swap() to properly handle ASOF JOIN MATCH_CONDITION commutation
+ return join.swap();
}
}
return null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java
index 459082926d3..899f5a3f511 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpPredicates.java
@@ -35,6 +35,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
@@ -252,14 +253,19 @@ public class PullUpPredicates extends PlanVisitor<ImmutableSet<Expression>, Void
predicates.addAll(join.getOtherJoinConjuncts());
}
break;
- case INNER_JOIN: {
+ case INNER_JOIN:
+ case ASOF_LEFT_INNER_JOIN:
+ case ASOF_RIGHT_INNER_JOIN: {
predicates.addAll(leftPredicates.get());
predicates.addAll(rightPredicates.get());
predicates.addAll(join.getHashJoinConjuncts());
- predicates.addAll(join.getOtherJoinConjuncts());
+ if (join.getJoinType() == JoinType.INNER_JOIN) {
+ predicates.addAll(join.getOtherJoinConjuncts());
+ }
break;
}
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
predicates.addAll(leftPredicates.get());
predicates.addAll(
generateNullTolerantPredicates(rightPredicates.get(), join.right().getOutputSet()));
@@ -271,6 +277,7 @@ public class PullUpPredicates extends PlanVisitor<ImmutableSet<Expression>, Void
break;
}
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
predicates.addAll(rightPredicates.get());
predicates.addAll(
generateNullTolerantPredicates(leftPredicates.get(), join.left().getOutputSet()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java
index cc2e6388a60..7ecb31201c8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderLimit.java
@@ -37,7 +37,8 @@ import java.util.stream.Collectors;
public class PullUpProjectUnderLimit extends OneRewriteRuleFactory {
@Override
public Rule build() {
- return logicalLimit(logicalProject(logicalJoin().when(j -> j.getJoinType().isLeftRightOuterOrCrossJoin()))
+ return logicalLimit(logicalProject(logicalJoin().when(j -> j.getJoinType().isLeftRightOuterOrCrossJoin()
+ || j.getJoinType().isAsofOuterJoin()))
.whenNot(p -> p.isAllSlots()))
.then(limit -> {
LogicalProject<LogicalJoin<Plan, Plan>> project = limit.child();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java
index 469444006d1..5e0dc47af6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderTopN.java
@@ -45,8 +45,9 @@ public class PullUpProjectUnderTopN extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalTopN(
- logicalProject(logicalJoin().when(j -> j.getJoinType().isLeftRightOuterOrCrossJoin()))
- .whenNot(p -> p.isAllSlots()))
+ logicalProject(logicalJoin().when(j -> j.getJoinType().isLeftRightOuterOrCrossJoin()
+ || j.getJoinType().isAsofOuterJoin()))
+ .whenNot(p -> p.isAllSlots()))
.then(topN -> {
LogicalProject<LogicalJoin<Plan, Plan>> project = topN.child();
Set<Slot> outputSet = project.child().getOutputSet();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java
index e717a00b3db..b16abf35250 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java
@@ -47,7 +47,9 @@ public class PushDownFilterThroughJoin extends OneRewriteRuleFactory {
JoinType.LEFT_SEMI_JOIN,
JoinType.LEFT_ANTI_JOIN,
JoinType.NULL_AWARE_LEFT_ANTI_JOIN,
- JoinType.CROSS_JOIN
+ JoinType.CROSS_JOIN,
+ JoinType.ASOF_LEFT_INNER_JOIN,
+ JoinType.ASOF_LEFT_OUTER_JOIN
);
public static final ImmutableList<JoinType> COULD_PUSH_THROUGH_RIGHT = ImmutableList.of(
@@ -55,7 +57,9 @@ public class PushDownFilterThroughJoin extends OneRewriteRuleFactory {
JoinType.RIGHT_OUTER_JOIN,
JoinType.RIGHT_SEMI_JOIN,
JoinType.RIGHT_ANTI_JOIN,
- JoinType.CROSS_JOIN
+ JoinType.CROSS_JOIN,
+ JoinType.ASOF_RIGHT_INNER_JOIN,
+ JoinType.ASOF_RIGHT_OUTER_JOIN
);
private static final ImmutableList<JoinType> COULD_PUSH_INSIDE = ImmutableList.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java
index 68cf03d7f5b..8aaeba36f32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimit.java
@@ -127,11 +127,13 @@ public class PushDownLimit implements RewriteRuleFactory {
private Plan pushLimitThroughJoin(LogicalLimit<? extends Plan> limit, LogicalJoin<Plan, Plan> join) {
switch (join.getJoinType()) {
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
return join.withChildren(
limit.withChildren(join.left()),
join.right()
);
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
return join.withChildren(
join.left(),
limit.withChildren(join.right())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java
index db538329339..906007decf4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoin.java
@@ -83,6 +83,7 @@ public class PushDownLimitDistinctThroughJoin implements RewriteRuleFactory {
.flatMap(e -> e.getInputSlots().stream()).collect(Collectors.toList());
switch (join.getJoinType()) {
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
if (!(join.left() instanceof Limit)
&& join.left().getOutputSet().containsAll(groupBySlots)
&& join.left().getOutputSet().equals(agg.getOutputSet())) {
@@ -91,6 +92,7 @@ public class PushDownLimitDistinctThroughJoin implements RewriteRuleFactory {
}
return null;
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
if (!(join.right() instanceof Limit) && join.right().getOutputSet().containsAll(groupBySlots)
&& join.right().getOutputSet().equals(agg.getOutputSet())) {
return join.withChildren(join.left(), limit.withLimitChild(limit.getLimit() + limit.getOffset(), 0,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java
index 4c6f426f7ac..4b0b8a89dc4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoin.java
@@ -98,7 +98,8 @@ public class PushDownTopNDistinctThroughJoin implements RewriteRuleFactory {
Set<Slot> groupBySlots = ((LogicalAggregate<?>) topN.child()).getGroupByExpressions().stream()
.flatMap(e -> e.getInputSlots().stream()).collect(Collectors.toSet());
switch (join.getJoinType()) {
- case LEFT_OUTER_JOIN: {
+ case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN: {
if (join.left() instanceof TopN) {
return null;
}
@@ -112,7 +113,8 @@ public class PushDownTopNDistinctThroughJoin implements RewriteRuleFactory {
}
return null;
}
- case RIGHT_OUTER_JOIN: {
+ case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN: {
if (join.right() instanceof TopN) {
return null;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java
index 7205784d025..74b9cb1f8e6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoin.java
@@ -94,6 +94,7 @@ public class PushDownTopNThroughJoin implements RewriteRuleFactory {
.flatMap(e -> e.getInputSlots().stream()).collect(Collectors.toList());
switch (join.getJoinType()) {
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
if (join.left() instanceof TopN) {
return null;
}
@@ -104,6 +105,7 @@ public class PushDownTopNThroughJoin implements RewriteRuleFactory {
}
return null;
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
if (join.right() instanceof TopN) {
return null;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java
index c1640671d33..d3ba907c5bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoin.java
@@ -46,8 +46,7 @@ public class PushFilterInsideJoin extends OneRewriteRuleFactory {
public Rule build() {
return logicalFilter(logicalJoin())
// TODO: current just handle cross/inner join.
- .when(filter -> filter.child().getJoinType().isCrossJoin()
- || filter.child().getJoinType().isInnerJoin())
+ .when(filter -> filter.child().getJoinType().isInnerOrCrossJoin())
.then(filter -> {
LogicalJoin<Plan, Plan> join = filter.child();
Set<Slot> childOutput = join.getOutputSet();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java
index f5bb7323155..6b3b540477c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java
@@ -85,7 +85,8 @@ public class ReorderJoin extends OneRewriteRuleFactory {
public Rule build() {
return logicalFilter(subTree(LogicalJoin.class, LogicalFilter.class))
.whenNot(filter -> filter.child() instanceof LogicalJoin
- && ((LogicalJoin<?, ?>) filter.child()).isMarkJoin())
+ && ((((LogicalJoin<?, ?>) filter.child()).isMarkJoin())
+ || ((LogicalJoin<?, ?>) filter.child()).getJoinType().isAsofJoin()))
.thenApply(ctx -> {
if (ctx.statementContext.getConnectContext().getSessionVariable().isDisableJoinReorder()
|| ctx.cascadesContext.isLeadingDisableJoinReorder()
@@ -162,7 +163,7 @@ public class ReorderJoin extends OneRewriteRuleFactory {
join = (LogicalJoin<?, ?>) plan;
}
- if (join.isMarkJoin()) {
+ if (join.isMarkJoin() || join.getJoinType().isAsofJoin()) {
return plan;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java
index dd04b5ceee8..98b82a3212a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SaltJoin.java
@@ -33,7 +33,6 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
@@ -50,7 +49,6 @@ import org.apache.doris.nereids.trees.plans.DistributeType;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -63,7 +61,6 @@ import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import org.jetbrains.annotations.Nullable;
import java.util.HashSet;
@@ -131,7 +128,8 @@ public class SaltJoin extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalJoin()
- .when(join -> join.getJoinType().isOneSideOuterJoin() || join.getJoinType().isInnerJoin())
+ .when(join -> join.getJoinType().isOneSideOuterJoin() || join.getJoinType().isInnerJoin()
+ || join.getJoinType().isAsofJoin())
.when(join -> join.getDistributeHint() != null && join.getDistributeHint().getSkewInfo() != null)
.whenNot(LogicalJoin::isMarkJoin)
.whenNot(join -> join.getDistributeHint().isSuccessInSkewRewrite())
@@ -158,9 +156,11 @@ public class SaltJoin extends OneRewriteRuleFactory {
if (!skewExpr.isSlot()) {
return null;
}
- if ((join.getJoinType().isLeftOuterJoin() || join.getJoinType().isInnerJoin())
+ if ((join.getJoinType().isLeftOuterJoin() || join.getJoinType().isInnerJoin()
+ || join.getJoinType().isAsofLeftOuterJoin() || join.getJoinType().isAsofInnerJoin())
&& !join.left().getOutput().contains((Slot) skewExpr)
- || join.getJoinType().isRightOuterJoin() && !join.right().getOutput().contains((Slot) skewExpr)) {
+ || (join.getJoinType().isRightOuterJoin()
+ || join.getJoinType().isAsofRightOuterJoin()) && !join.right().getOutput().contains((Slot) skewExpr)) {
return null;
}
int factor = getSaltFactor();
@@ -205,12 +205,16 @@ public class SaltJoin extends OneRewriteRuleFactory {
switch (join.getJoinType()) {
case INNER_JOIN:
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_INNER_JOIN:
+ case ASOF_RIGHT_INNER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
leftProject = addRandomSlot(leftSkewExpr, skewSideValues, join.left(), factor, type,
statementContext);
rightProject = expandSkewValueRows(rightSkewExpr, expandSideValues, join.right(), factor, type,
statementContext);
break;
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
leftProject = expandSkewValueRows(leftSkewExpr, expandSideValues, join.left(), factor, type,
statementContext);
rightProject = addRandomSlot(rightSkewExpr, skewSideValues, join.right(), factor, type,
@@ -364,7 +368,7 @@ public class SaltJoin extends OneRewriteRuleFactory {
if (skewConjunct instanceof NullSafeEqual) {
return Utils.fastToImmutableList(skewValuesSet);
} else if (skewConjunct instanceof EqualTo) {
- if (join.getJoinType().isInnerJoin()) {
+ if (join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin()) {
return skewValuesSet.stream().filter(value -> !(value instanceof NullLiteral))
.collect(ImmutableList.toImmutableList());
} else {
@@ -373,37 +377,4 @@ public class SaltJoin extends OneRewriteRuleFactory {
}
return ImmutableList.of();
}
-
- private static LogicalJoin<Plan, Plan> addNotNull(LogicalJoin<Plan, Plan> join, Expression skewConjunct,
- Set<Expression> skewValuesSet) {
- if (skewConjunct instanceof NullSafeEqual) {
- return join;
- }
- boolean containsNull = skewValuesSet.stream().anyMatch(value -> value instanceof NullLiteral);
- if (!containsNull) {
- return join;
- }
-
- LogicalFilter<Plan> leftFilter =
- new LogicalFilter<>(ImmutableSet.of(new Not(new IsNull(skewConjunct.child(0)))), join.left());
- LogicalFilter<Plan> rightFilter =
- new LogicalFilter<>(ImmutableSet.of(new Not(new IsNull(skewConjunct.child(1)))), join.right());
- DistributeHint hint = join.getDistributeHint();
- switch (join.getJoinType()) {
- case INNER_JOIN:
- hint.setStatus(HintStatus.SUCCESS);
- hint.setSkewInfo(hint.getSkewInfo().withSuccessInSaltJoin(true));
- return join.withDistributeHintChildren(hint, leftFilter, rightFilter);
- case LEFT_OUTER_JOIN:
- hint.setStatus(HintStatus.SUCCESS);
- hint.setSkewInfo(hint.getSkewInfo().withSuccessInSaltJoin(true));
- return join.withDistributeHintChildren(hint, join.left(), rightFilter);
- case RIGHT_OUTER_JOIN:
- hint.setStatus(HintStatus.SUCCESS);
- hint.setSkewInfo(hint.getSkewInfo().withSuccessInSaltJoin(true));
- return join.withDistributeHintChildren(hint, leftFilter, join.right());
- default:
- return join;
- }
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SkewJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SkewJoin.java
index f017ec99a19..60c6c3ee756 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SkewJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SkewJoin.java
@@ -50,7 +50,7 @@ public class SkewJoin extends OneRewriteRuleFactory {
public Rule build() {
return logicalJoin()
.when(join -> join.getJoinType().isOneSideOuterJoin()
- || join.getJoinType().isInnerJoin())
+ || join.getJoinType().isInnerJoin() || join.getJoinType().isAsofJoin())
.when(join -> join.getDistributeHint().distributeType == DistributeType.NONE)
.whenNot(LogicalJoin::isMarkJoin)
.thenApply(SkewJoin::transform).toRule(RuleType.SALT_JOIN);
@@ -81,14 +81,15 @@ public class SkewJoin extends OneRewriteRuleFactory {
if (join.left().getOutputSet().contains(equal.right())) {
equal = equal.commute();
}
- if (join.getJoinType().isInnerJoin() || join.getJoinType().isLeftOuterJoin()) {
+ if (join.getJoinType().isInnerJoin() || join.getJoinType().isLeftOuterJoin()
+ || join.getJoinType().isAsofInnerJoin() || join.getJoinType().isAsofLeftOuterJoin()) {
Expression leftEqHand = equal.child(0);
if (left.getStats().findColumnStatistics(leftEqHand) != null
&& left.getStats().findColumnStatistics(leftEqHand).getHotValues() != null) {
skewExpr = leftEqHand;
hotValues.addAll(left.getStats().findColumnStatistics(leftEqHand).getHotValues().keySet());
}
- } else if (join.getJoinType().isRightOuterJoin()) {
+ } else if (join.getJoinType().isRightOuterJoin() || join.getJoinType().isAsofRightOuterJoin()) {
Expression rightEqHand = equal.child(1);
if (right.getStats().findColumnStatistics(rightEqHand) != null
&& right.getStats().findColumnStatistics(rightEqHand).getHotValues() != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java
index 7ee1dcf94f9..509ffa8bb0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoin.java
@@ -21,7 +21,6 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.TransposeSemiJoinLogicalJoinProject.ContainsType;
import org.apache.doris.nereids.trees.expressions.ExprId;
-import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.qe.ConnectContext;
@@ -66,7 +65,7 @@ public class TransposeSemiJoinLogicalJoin extends OneRewriteRuleFactory {
* A B A C
*/
// RIGHT_OUTER_JOIN should be eliminated in rewrite phase
- if (bottomJoin.getJoinType() == JoinType.RIGHT_OUTER_JOIN) {
+ if (bottomJoin.getJoinType().isRightOuterJoin()) {
return null;
}
Plan newBottomSemiJoin = topSemiJoin.withChildren(a, c);
@@ -80,7 +79,7 @@ public class TransposeSemiJoinLogicalJoin extends OneRewriteRuleFactory {
* A B B C
*/
// LEFT_OUTER_JOIN should be eliminated in rewrite phase
- if (bottomJoin.getJoinType() == JoinType.LEFT_OUTER_JOIN) {
+ if (bottomJoin.getJoinType().isLeftOuterJoin()) {
return null;
}
Plan newBottomSemiJoin = topSemiJoin.withChildren(b, c);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java
index 95065e1b33d..40e49e13bcd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/TransposeSemiJoinLogicalJoinProject.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
-import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -80,7 +79,7 @@ public class TransposeSemiJoinLogicalJoinProject extends OneRewriteRuleFactory {
// RIGHT_OUTER_JOIN should be eliminated in rewrite phase
// TODO: when top join is ANTI JOIN, bottomJoin may be RIGHT_OUTER_JOIN
// Can we also do the transformation?
- if (bottomJoin.getJoinType() == JoinType.RIGHT_OUTER_JOIN) {
+ if (bottomJoin.getJoinType().isRightOuterJoin()) {
return null;
}
@@ -100,7 +99,7 @@ public class TransposeSemiJoinLogicalJoinProject extends OneRewriteRuleFactory {
// LEFT_OUTER_JOIN should be eliminated in rewrite phase
// TODO: when top join is ANTI JOIN, bottomJoin may be RIGHT_OUTER_JOIN
// Can we also do the transformation?
- if (bottomJoin.getJoinType() == JoinType.LEFT_OUTER_JOIN) {
+ if (bottomJoin.getJoinType().isLeftOuterJoin()) {
return null;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
index 0e03a6cf7bd..84d8b334fdc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/JoinEstimation.java
@@ -359,6 +359,83 @@ public class JoinEstimation {
}
}
+ private static Statistics estimateAsofInnerJoin(Statistics leftStats, Statistics rightStats,
+ Statistics innerJoinStats, Join join) {
+ if (joinConditionContainsUnknownColumnStats(leftStats, rightStats, join)) {
+ double sel = computeSelectivityForBuildSideWhenColStatsUnknown(rightStats, join);
+ Statistics result;
+ if (join.getJoinType().isAsofLeftInnerJoin()) {
+ result = new StatisticsBuilder().setRowCount(leftStats.getRowCount() * sel)
+ .putColumnStatistics(leftStats.columnStatistics())
+ .putColumnStatistics(rightStats.columnStatistics())
+ .build();
+ } else {
+ //asof right inner join
+ result = new StatisticsBuilder().setRowCount(rightStats.getRowCount() * sel)
+ .putColumnStatistics(leftStats.columnStatistics())
+ .putColumnStatistics(rightStats.columnStatistics())
+ .build();
+ }
+ result.normalizeColumnStatistics();
+ return result;
+ }
+ double rowCount = Double.POSITIVE_INFINITY;
+ for (Expression conjunct : join.getEqualPredicates()) {
+ double eqRowCount = estimateAsofInnerJoinCountBySlotsEqual(leftStats, rightStats,
+ join, (EqualPredicate) conjunct);
+ if (rowCount > eqRowCount) {
+ rowCount = eqRowCount;
+ }
+ }
+ if (Double.isInfinite(rowCount)) {
+ //slotsEqual estimation failed, fall back to original algorithm
+ double baseRowCount =
+ join.getJoinType().isAsofLeftInnerJoin() ? leftStats.getRowCount() : rightStats.getRowCount();
+ rowCount = Math.min(innerJoinStats.getRowCount(), baseRowCount);
+ return innerJoinStats.withRowCountAndEnforceValid(rowCount);
+ } else {
+ StatisticsBuilder builder;
+ if (join.getJoinType().isAsofLeftInnerJoin()) {
+ builder = new StatisticsBuilder(leftStats);
+ } else {
+ //asof right inner join
+ builder = new StatisticsBuilder(rightStats);
+ }
+ builder.setRowCount(rowCount);
+ Statistics outputStats = builder.build();
+ outputStats.normalizeColumnStatistics();
+ return outputStats;
+ }
+ }
+
+ private static double estimateAsofInnerJoinCountBySlotsEqual(Statistics leftStats,
+ Statistics rightStats, Join join, EqualPredicate equalTo) {
+ Expression eqLeft = equalTo.left();
+ Expression eqRight = equalTo.right();
+ ColumnStatistic probColStats = leftStats.findColumnStatistics(eqLeft);
+ ColumnStatistic buildColStats;
+ if (probColStats == null) {
+ probColStats = leftStats.findColumnStatistics(eqRight);
+ buildColStats = rightStats.findColumnStatistics(eqLeft);
+ } else {
+ buildColStats = rightStats.findColumnStatistics(eqRight);
+ }
+ if (probColStats == null || buildColStats == null) {
+ return Double.POSITIVE_INFINITY;
+ }
+
+ double rowCount;
+ if (join.getJoinType().isAsofLeftInnerJoin()) {
+ rowCount = StatsMathUtil.divide(leftStats.getRowCount() * buildColStats.ndv,
+ buildColStats.getOriginalNdv());
+ } else {
+ // asof right inner join
+ rowCount = StatsMathUtil.divide(rightStats.getRowCount() * probColStats.ndv,
+ probColStats.getOriginalNdv());
+ }
+ return Math.max(1, rowCount);
+ }
+
/**
* outer join generates nulls.
* for example, T1 left outer join T2,
@@ -418,6 +495,20 @@ public class JoinEstimation {
updateNumNullsForOuterJoin(crossJoinStats, innerJoinStats, rightStats, leftStats, rowCount);
updateJoinConditionColumnStatistics(crossJoinStats, join);
return crossJoinStats.withRowCountAndEnforceValid(rowCount);
+ } else if (joinType == JoinType.ASOF_LEFT_OUTER_JOIN) {
+ double rowCount = Math.max(leftStats.getRowCount(), 1);
+ updateNumNullsForOuterJoin(crossJoinStats, innerJoinStats, leftStats, rightStats, rowCount);
+ updateJoinConditionColumnStatistics(crossJoinStats, join);
+ return crossJoinStats.withRowCountAndEnforceValid(rowCount);
+ } else if (joinType == JoinType.ASOF_RIGHT_OUTER_JOIN) {
+ double rowCount = Math.max(rightStats.getRowCount(), 1);
+ updateNumNullsForOuterJoin(crossJoinStats, innerJoinStats, rightStats, leftStats, rowCount);
+ updateJoinConditionColumnStatistics(crossJoinStats, join);
+ return crossJoinStats.withRowCountAndEnforceValid(rowCount);
+ } else if (joinType.isAsofInnerJoin()) {
+ Statistics outputStats = estimateAsofInnerJoin(leftStats, rightStats, innerJoinStats, join);
+ updateJoinConditionColumnStatistics(outputStats, join);
+ return outputStats;
} else if (joinType == JoinType.CROSS_JOIN) {
updateJoinConditionColumnStatistics(crossJoinStats, join);
return crossJoinStats;
@@ -443,7 +534,7 @@ public class JoinEstimation {
if (eqRight instanceof Cast) {
eqRight = eqRight.child(0);
}
- if (joinType == JoinType.INNER_JOIN) {
+ if (joinType.isInnerJoin() || joinType.isAsofInnerJoin()) {
ColumnStatisticBuilder builder = new ColumnStatisticBuilder(leftColStats);
builder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv));
// update hot values
@@ -463,7 +554,7 @@ public class JoinEstimation {
}
updatedCols.put(eqLeft, builder.build());
updatedCols.put(eqRight, builder.build());
- } else if (joinType == JoinType.LEFT_OUTER_JOIN) {
+ } else if (joinType.isLeftOuterJoin() || joinType.isAsofLeftOuterJoin()) {
ColumnStatisticBuilder rightBuilder = new ColumnStatisticBuilder(rightColStats);
rightBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv));
// update hot values
@@ -482,13 +573,11 @@ public class JoinEstimation {
}
}
updatedCols.put(eqRight, rightBuilder.build());
- } else if (joinType == JoinType.LEFT_SEMI_JOIN
- || joinType == JoinType.LEFT_ANTI_JOIN
- || joinType == JoinType.NULL_AWARE_LEFT_ANTI_JOIN) {
+ } else if (joinType.isLeftSemiOrAntiJoin()) {
ColumnStatisticBuilder leftBuilder = new ColumnStatisticBuilder(leftColStats);
leftBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv));
updatedCols.put(eqLeft, leftBuilder.build());
- } else if (joinType == JoinType.RIGHT_OUTER_JOIN) {
+ } else if (joinType.isRightOuterJoin() || joinType.isAsofRightOuterJoin()) {
ColumnStatisticBuilder leftBuilder = new ColumnStatisticBuilder(leftColStats);
leftBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv));
// update hot values
@@ -507,12 +596,11 @@ public class JoinEstimation {
}
}
updatedCols.put(eqLeft, leftBuilder.build());
- } else if (joinType == JoinType.RIGHT_SEMI_JOIN
- || joinType == JoinType.RIGHT_ANTI_JOIN) {
+ } else if (joinType.isRightSemiOrAntiJoin()) {
ColumnStatisticBuilder rightBuilder = new ColumnStatisticBuilder(rightColStats);
rightBuilder.setNdv(Math.min(leftColStats.ndv, rightColStats.ndv));
updatedCols.put(eqRight, rightBuilder.build());
- } else if (joinType == JoinType.FULL_OUTER_JOIN || joinType == JoinType.CROSS_JOIN) {
+ } else if (joinType.isFullOuterJoin() || joinType.isCrossJoin()) {
// ignore
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
index 112c1d98a98..b1fcf9bd460 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java
@@ -39,6 +39,10 @@ public enum JoinType {
RIGHT_ANTI_JOIN,
CROSS_JOIN,
NULL_AWARE_LEFT_ANTI_JOIN,
+ ASOF_LEFT_INNER_JOIN,
+ ASOF_RIGHT_INNER_JOIN,
+ ASOF_LEFT_OUTER_JOIN,
+ ASOF_RIGHT_OUTER_JOIN
;
private static final Map<JoinType, JoinType> joinSwapMap = ImmutableMap
@@ -52,6 +56,10 @@ public enum JoinType {
.put(RIGHT_OUTER_JOIN, LEFT_OUTER_JOIN)
.put(LEFT_ANTI_JOIN, RIGHT_ANTI_JOIN)
.put(RIGHT_ANTI_JOIN, LEFT_ANTI_JOIN)
+ .put(ASOF_LEFT_INNER_JOIN, ASOF_RIGHT_INNER_JOIN)
+ .put(ASOF_RIGHT_INNER_JOIN, ASOF_LEFT_INNER_JOIN)
+ .put(ASOF_LEFT_OUTER_JOIN, ASOF_RIGHT_OUTER_JOIN)
+ .put(ASOF_RIGHT_OUTER_JOIN, ASOF_LEFT_OUTER_JOIN)
.build();
// TODO: the right-semi/right-anti/right-outer join is not derived in paper. We need to derive them
@@ -112,6 +120,14 @@ public enum JoinType {
return JoinOperator.LEFT_SEMI_JOIN;
case RIGHT_SEMI_JOIN:
return JoinOperator.RIGHT_SEMI_JOIN;
+ case ASOF_LEFT_INNER_JOIN:
+ return JoinOperator.ASOF_LEFT_INNER_JOIN;
+ case ASOF_RIGHT_INNER_JOIN:
+ return JoinOperator.ASOF_RIGHT_INNER_JOIN;
+ case ASOF_LEFT_OUTER_JOIN:
+ return JoinOperator.ASOF_LEFT_OUTER_JOIN;
+ case ASOF_RIGHT_OUTER_JOIN:
+ return JoinOperator.ASOF_RIGHT_OUTER_JOIN;
case CROSS_JOIN:
return JoinOperator.CROSS_JOIN;
default:
@@ -213,6 +229,43 @@ public enum JoinType {
return this == NULL_AWARE_LEFT_ANTI_JOIN;
}
+ public final boolean isAsofLeftInnerJoin() {
+ return this == ASOF_LEFT_INNER_JOIN;
+ }
+
+ public final boolean isAsofRightInnerJoin() {
+ return this == ASOF_RIGHT_INNER_JOIN;
+ }
+
+ public final boolean isAsofInnerJoin() {
+ return this == ASOF_LEFT_INNER_JOIN || this == ASOF_RIGHT_INNER_JOIN;
+ }
+
+ public final boolean isAsofLeftOuterJoin() {
+ return this == ASOF_LEFT_OUTER_JOIN;
+ }
+
+ public final boolean isAsofRightOuterJoin() {
+ return this == ASOF_RIGHT_OUTER_JOIN;
+ }
+
+ public final boolean isAsofOuterJoin() {
+ return this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN;
+ }
+
+ public final boolean isAsofLeftJoin() {
+ return this == ASOF_LEFT_INNER_JOIN || this == ASOF_LEFT_OUTER_JOIN;
+ }
+
+ public final boolean isAsofRightJoin() {
+ return this == ASOF_RIGHT_INNER_JOIN || this == ASOF_RIGHT_OUTER_JOIN;
+ }
+
+ public final boolean isAsofJoin() {
+ return this == ASOF_LEFT_INNER_JOIN || this == ASOF_RIGHT_INNER_JOIN
+ || this == ASOF_LEFT_OUTER_JOIN || this == ASOF_RIGHT_OUTER_JOIN;
+ }
+
public final boolean isSwapJoinType() {
return joinSwapMap.containsKey(this);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
index f7bf44b5518..b1ccabc52db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DataTrait.Builder;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext;
+import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.ExprId;
@@ -185,9 +186,32 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
this.exceptAsteriskOutputs = exceptAsteriskOutputs;
}
+ /**
+ * Swap the children of this join. For ASOF JOIN, the MATCH_CONDITION (in otherJoinConjuncts) must be commuted
+ * when swapping join children. E.g., "A.ts >= B.ts" becomes "B.ts <= A.ts"
+ * This is because BE assumes MATCH_CONDITION has probe column on left and build column on right
+ */
public LogicalJoin<? extends Plan, ? extends Plan> swap() {
- return withTypeChildren(getJoinType().swap(),
- right(), left(), null);
+ JoinType swappedType = getJoinType().swap();
+ List<Expression> swappedOtherConjuncts = otherJoinConjuncts;
+
+ // For ASOF JOIN, the MATCH_CONDITION (in otherJoinConjuncts) must be commuted
+ // when swapping join children. E.g., "A.ts >= B.ts" becomes "B.ts <= A.ts"
+ // This is because BE assumes MATCH_CONDITION has probe column on left and build column on right
+ if (getJoinType().isAsofJoin() && !otherJoinConjuncts.isEmpty()) {
+ swappedOtherConjuncts = otherJoinConjuncts.stream()
+ .map(expr -> {
+ if (expr instanceof ComparisonPredicate) {
+ return ((ComparisonPredicate) expr).commute();
+ }
+ return expr;
+ })
+ .collect(ImmutableList.toImmutableList());
+ }
+
+ return new LogicalJoin<>(swappedType, hashJoinConjuncts, swappedOtherConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, exceptAsteriskOutputs,
+ Optional.empty(), Optional.empty(), ImmutableList.of(right(), left()), null);
}
public List<Expression> getOtherJoinConjuncts() {
@@ -508,6 +532,14 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
Optional.empty(), Optional.empty(), ImmutableList.of(left, right), otherJoinReorderContext);
}
+ public LogicalJoin<Plan, Plan> withTypeChildrenAndOtherConjuncts(JoinType joinType, Plan left, Plan right,
+ List<Expression> otherJoinConjuncts,
+ JoinReorderContext otherJoinReorderContext) {
+ return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
+ hint, markJoinSlotReference, exceptAsteriskOutputs,
+ Optional.empty(), Optional.empty(), ImmutableList.of(left, right), otherJoinReorderContext);
+ }
+
public LogicalJoin<Plan, Plan> withDistributeHintChildren(DistributeHint hint, Plan left, Plan right) {
return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinConjuncts,
hint, markJoinSlotReference, exceptAsteriskOutputs,
@@ -549,7 +581,7 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
// this function is only used by EliminateJoinByFK rule, and EliminateJoinByFK is disabled for mark join
// so markJoinConjuncts is not processed now
// TODO: Use fd in the future
- if (!joinType.isInnerJoin() && !joinType.isSemiJoin()) {
+ if (!joinType.isInnerJoin() && !joinType.isSemiJoin() && !joinType.isAsofInnerJoin()) {
return ImmutableEqualSet.empty();
}
ImmutableEqualSet.Builder<Slot> builder = new ImmutableEqualSet.Builder<>();
@@ -582,9 +614,9 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
// TODO disable function dependence calculation for mark join, but need re-think this in future.
return;
}
- if (joinType.isLeftSemiOrAntiJoin()) {
+ if (joinType.isLeftSemiOrAntiJoin() || joinType.isAsofLeftJoin()) {
builder.addUniqueSlot(left().getLogicalProperties().getTrait());
- } else if (joinType.isRightSemiOrAntiJoin()) {
+ } else if (joinType.isRightSemiOrAntiJoin() || joinType.isAsofRightJoin()) {
builder.addUniqueSlot(right().getLogicalProperties().getTrait());
}
// if there is non-equal join conditions, don't propagate unique
@@ -627,6 +659,8 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
}
switch (joinType) {
case INNER_JOIN:
+ case ASOF_LEFT_INNER_JOIN:
+ case ASOF_RIGHT_INNER_JOIN:
case CROSS_JOIN:
builder.addUniformSlot(left().getLogicalProperties().getTrait());
builder.addUniformSlot(right().getLogicalProperties().getTrait());
@@ -641,10 +675,12 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
builder.addUniformSlot(right().getLogicalProperties().getTrait());
break;
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
builder.addUniformSlot(left().getLogicalProperties().getTrait());
builder.addUniformSlotForOuterJoinNullableSide(right().getLogicalProperties().getTrait());
break;
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
builder.addUniformSlot(right().getLogicalProperties().getTrait());
builder.addUniformSlotForOuterJoinNullableSide(left().getLogicalProperties().getTrait());
break;
@@ -665,7 +701,7 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
if (!joinType.isRightSemiOrAntiJoin()) {
builder.addEqualSet(left().getLogicalProperties().getTrait());
}
- if (joinType.isInnerJoin()) {
+ if (joinType.isInnerJoin() || joinType.isAsofInnerJoin()) {
for (Expression expression : getHashJoinConjuncts()) {
Optional<Pair<Slot, Slot>> equalSlot = ExpressionUtils.extractEqualSlot(expression);
equalSlot.ifPresent(slotSlotPair -> builder.addEqualPair(slotSlotPair.first, slotSlotPair.second));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java
index 0b056da686c..a6b685a782a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalUsingJoin.java
@@ -49,21 +49,23 @@ public class LogicalUsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE ext
private final JoinType joinType;
private final ImmutableList<Expression> usingSlots;
private final DistributeHint hint;
+ private final Optional<Expression> matchCondition;
public LogicalUsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild,
- List<Expression> usingSlots, DistributeHint hint) {
- this(joinType, leftChild, rightChild, usingSlots, Optional.empty(), Optional.empty(), hint);
+ List<Expression> usingSlots, Optional<Expression> matchCondition, DistributeHint hint) {
+ this(joinType, leftChild, rightChild, usingSlots, matchCondition, Optional.empty(), Optional.empty(), hint);
}
/**
* Constructor.
*/
public LogicalUsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild,
- List<Expression> usingSlots, Optional<GroupExpression> groupExpression,
+ List<Expression> usingSlots, Optional<Expression> matchCondition, Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, DistributeHint hint) {
super(PlanType.LOGICAL_USING_JOIN, groupExpression, logicalProperties, leftChild, rightChild);
this.joinType = joinType;
this.usingSlots = ImmutableList.copyOf(usingSlots);
+ this.matchCondition = matchCondition;
this.hint = hint;
}
@@ -75,20 +77,20 @@ public class LogicalUsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE ext
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalUsingJoin<>(joinType, child(0), child(1),
- usingSlots, groupExpression, Optional.of(getLogicalProperties()), hint);
+ usingSlots, matchCondition, groupExpression, Optional.of(getLogicalProperties()), hint);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalUsingJoin<>(joinType, children.get(0), children.get(1),
- usingSlots, groupExpression, logicalProperties, hint);
+ usingSlots, matchCondition, groupExpression, logicalProperties, hint);
}
@Override
public Plan withChildren(List<Plan> children) {
return new LogicalUsingJoin<>(joinType, children.get(0), children.get(1),
- usingSlots, groupExpression, Optional.of(getLogicalProperties()), hint);
+ usingSlots, matchCondition, groupExpression, Optional.of(getLogicalProperties()), hint);
}
@Override
@@ -117,6 +119,10 @@ public class LogicalUsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE ext
return ExpressionUtils.EMPTY_CONDITION;
}
+ public Optional<Expression> getMatchCondition() {
+ return matchCondition;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -128,12 +134,13 @@ public class LogicalUsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE ext
LogicalUsingJoin<?, ?> usingJoin = (LogicalUsingJoin<?, ?>) o;
return joinType == usingJoin.joinType
&& Objects.equals(usingSlots, usingJoin.usingSlots)
+ && Objects.equals(matchCondition, usingJoin.matchCondition)
&& Objects.equals(hint, usingJoin.hint);
}
@Override
public int hashCode() {
- return Objects.hash(joinType, usingSlots, hint);
+ return Objects.hash(joinType, usingSlots, matchCondition, hint);
}
@Override
@@ -141,6 +148,7 @@ public class LogicalUsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE ext
List<Object> args = Lists.newArrayList(
"type", joinType,
"usingSlots", usingSlots,
+ "matchCondition", matchCondition,
"stats", statistics);
if (hint.distributeType != DistributeType.NONE) {
args.add("hint");
@@ -155,6 +163,11 @@ public class LogicalUsingJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE ext
sb.append(left().toDigest());
sb.append(" ").append(joinType).append(" ");
sb.append(right().toDigest());
+ if (matchCondition.isPresent()) {
+ sb.append("MATCH_CONDITION (");
+ sb.append(matchCondition.get().toDigest());
+ sb.append(")");
+ }
sb.append(" USING (");
sb.append(
usingSlots.stream().map(Expression::toDigest)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
index ca84e1afc2f..9145b6bfbf4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
@@ -289,11 +289,11 @@ public class PhysicalHashJoin<
// left/right outer join propagate left/right uniforms slots
// And if the right/left hash keys is unique,
// join can propagate left/right functional dependencies
- if (joinType.isLeftOuterJoin() && isRightUnique) {
+ if ((joinType.isLeftOuterJoin() || joinType.isAsofLeftOuterJoin()) && isRightUnique) {
builder.addUniqueSlot(left().getLogicalProperties().getTrait());
- } else if (joinType.isRightOuterJoin() && isLeftUnique) {
+ } else if ((joinType.isRightOuterJoin() || joinType.isAsofRightOuterJoin()) && isLeftUnique) {
builder.addUniqueSlot(right().getLogicalProperties().getTrait());
- } else if (joinType.isInnerJoin() && isLeftUnique && isRightUnique) {
+ } else if ((joinType.isInnerJoin() || joinType.isAsofInnerJoin()) && isLeftUnique && isRightUnique) {
// inner join propagate uniforms slots
// And if the hash keys is unique, inner join can propagate all functional dependencies
builder.addDataTrait(left().getLogicalProperties().getTrait());
@@ -347,7 +347,7 @@ public class PhysicalHashJoin<
if (!joinType.isRightSemiOrAntiJoin()) {
builder.addEqualSet(left().getLogicalProperties().getTrait());
}
- if (joinType.isInnerJoin()) {
+ if (joinType.isInnerJoin() || joinType.isAsofInnerJoin()) {
for (Expression expression : getHashJoinConjuncts()) {
Optional<Pair<Slot, Slot>> equalSlot = ExpressionUtils.extractEqualSlot(expression);
equalSlot.ifPresent(slotSlotPair -> builder.addEqualPair(slotSlotPair.first, slotSlotPair.second));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
index 745dc2734b4..8812d034b65 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
@@ -80,7 +80,8 @@ public class JoinUtils {
}
public static boolean couldBroadcast(Join join) {
- return !(join.getJoinType().isRightJoin() || join.getJoinType().isFullOuterJoin());
+ return !(join.getJoinType().isRightJoin() || join.getJoinType().isFullOuterJoin()
+ || join.getJoinType().isAsofRightOuterJoin());
}
/**
@@ -377,7 +378,8 @@ public class JoinUtils {
* Check whether the given join can be eliminated by pk-fk
*/
public static boolean canEliminateByFk(LogicalJoin<?, ?> join, Plan primaryPlan, Plan foreignPlan) {
- if (!join.getJoinType().isInnerJoin() || !join.getOtherJoinConjuncts().isEmpty() || join.isMarkJoin()) {
+ if (!(join.getJoinType().isInnerJoin() || join.getJoinType().isAsofInnerJoin())
+ || !join.getOtherJoinConjuncts().isEmpty() || join.isMarkJoin()) {
return false;
}
@@ -400,7 +402,7 @@ public class JoinUtils {
* can this join be eliminated by its left child
*/
public static boolean canEliminateByLeft(LogicalJoin<?, ?> join, DataTrait rightFuncDeps) {
- if (join.getJoinType().isLeftOuterJoin()) {
+ if (join.getJoinType().isLeftOuterJoin() || join.getJoinType().isAsofLeftOuterJoin()) {
Pair<Set<Slot>, Set<Slot>> njHashKeys = join.extractNullRejectHashKeys();
if (!join.getOtherJoinConjuncts().isEmpty() || njHashKeys == null) {
return false;
@@ -444,11 +446,13 @@ public class JoinUtils {
case RIGHT_ANTI_JOIN:
return ImmutableList.copyOf(rightOutput);
case LEFT_OUTER_JOIN:
+ case ASOF_LEFT_OUTER_JOIN:
return ImmutableList.<Slot>builder()
.addAll(leftOutput)
.addAll(applyNullable(rightOutput, true))
.build();
case RIGHT_OUTER_JOIN:
+ case ASOF_RIGHT_OUTER_JOIN:
return ImmutableList.<Slot>builder()
.addAll(applyNullable(leftOutput, true))
.addAll(rightOutput)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index 81f8e436d46..761f081eaa1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -101,10 +101,14 @@ public class HashJoinNode extends JoinNodeBase {
}
this.distrMode = DistributionMode.NONE;
this.otherJoinConjuncts = otherJoinConjuncts;
+ this.matchCondition = null;
}
+ private final Expr matchCondition;
+
public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp,
- List<Expr> eqJoinConjuncts, List<Expr> otherJoinConjuncts, List<Expr> markJoinConjuncts,
+ List<Expr> eqJoinConjuncts, List<Expr> otherJoinConjuncts, Expr matchCondition,
+ List<Expr> markJoinConjuncts,
List<Expr> srcToOutputList, TupleDescriptor intermediateTuple,
TupleDescriptor outputTuple, boolean isMarkJoin) {
super(id, "HASH JOIN", StatisticalType.HASH_JOIN_NODE, joinOp, isMarkJoin);
@@ -131,6 +135,7 @@ public class HashJoinNode extends JoinNodeBase {
}
this.distrMode = DistributionMode.NONE;
this.otherJoinConjuncts = otherJoinConjuncts;
+ this.matchCondition = matchCondition;
this.markJoinConjuncts = markJoinConjuncts;
children.add(outer);
children.add(inner);
@@ -216,6 +221,11 @@ public class HashJoinNode extends JoinNodeBase {
for (Expr e : otherJoinConjuncts) {
msg.hash_join_node.addToOtherJoinConjuncts(e.treeToThrift());
}
+ if (matchCondition != null) {
+ Preconditions.checkState(joinOp == JoinOperator.ASOF_LEFT_OUTER_JOIN
+ || joinOp == JoinOperator.ASOF_LEFT_INNER_JOIN, "match condition is not allowed in " + joinOp);
+ msg.hash_join_node.setMatchCondition(matchCondition.treeToThrift());
+ }
if (markJoinConjuncts != null) {
if (eqJoinConjuncts.isEmpty()) {
@@ -276,6 +286,10 @@ public class HashJoinNode extends JoinNodeBase {
output.append(detailPrefix).append("other join predicates: ")
.append(getExplainString(otherJoinConjuncts)).append("\n");
}
+ if (matchCondition != null) {
+ output.append(detailPrefix).append("match condition: ")
+ .append(matchCondition.toSql()).append("\n");
+ }
if (markJoinConjuncts != null && !markJoinConjuncts.isEmpty()) {
output.append(detailPrefix).append("mark join predicates: ")
.append(getExplainString(markJoinConjuncts)).append("\n");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
index 6c5bfbf8ea0..b331681819e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java
@@ -65,7 +65,9 @@ public class NestedLoopJoinNode extends JoinNodeBase {
public static boolean canParallelize(JoinOperator joinOp) {
return joinOp == JoinOperator.CROSS_JOIN || joinOp == JoinOperator.INNER_JOIN
|| joinOp == JoinOperator.LEFT_OUTER_JOIN || joinOp == JoinOperator.LEFT_SEMI_JOIN
- || joinOp == JoinOperator.LEFT_ANTI_JOIN || joinOp == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
+ || joinOp == JoinOperator.LEFT_ANTI_JOIN || joinOp == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN
+ || joinOp == JoinOperator.ASOF_LEFT_INNER_JOIN || joinOp == JoinOperator.ASOF_RIGHT_INNER_JOIN
+ || joinOp == JoinOperator.ASOF_LEFT_OUTER_JOIN;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 6be5527341b..7ffb82eef54 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -173,6 +173,7 @@ class StructInfoMapTest extends SqlTestBase {
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true;
+ connectContext.getSessionVariable().materializedViewRewriteDurationThresholdMs = 1000000;
dropMvByNereids("drop materialized view if exists mv1");
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
@@ -198,8 +199,8 @@ class StructInfoMapTest extends SqlTestBase {
Multimap<Integer, Integer> commonTableIdToRelationIdMap
= c1.getStatementContext().getCommonTableIdToRelationIdMap();
BitSet targetBitSet = new BitSet();
- for (Integer relationId : commonTableIdToRelationIdMap.values()) {
- targetBitSet.set(relationId);
+ for (Integer tableId : commonTableIdToRelationIdMap.keys()) {
+ targetBitSet.set(tableId);
}
c1.getMemo().incrementAndGetRefreshVersion(targetBitSet);
int memoVersion = StructInfoMap.getMemoVersion(targetBitSet, c1.getMemo().getRefreshVersion());
@@ -241,6 +242,7 @@ class StructInfoMapTest extends SqlTestBase {
};
connectContext.getSessionVariable().enableMaterializedViewRewrite = true;
connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true;
+ connectContext.getSessionVariable().materializedViewRewriteDurationThresholdMs = 1000000;
dropMvByNereids("drop materialized view if exists mv1");
createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n"
+ " DISTRIBUTED BY RANDOM BUCKETS 1\n"
@@ -265,8 +267,8 @@ class StructInfoMapTest extends SqlTestBase {
Multimap<Integer, Integer> commonTableIdToRelationIdMap
= c1.getStatementContext().getCommonTableIdToRelationIdMap();
BitSet targetBitSet = new BitSet();
- for (Integer relationId : commonTableIdToRelationIdMap.values()) {
- targetBitSet.set(relationId);
+ for (Integer tableId : commonTableIdToRelationIdMap.keys()) {
+ targetBitSet.set(tableId);
}
c1.getMemo().incrementAndGetRefreshVersion(targetBitSet);
int memoVersion = StructInfoMap.getMemoVersion(targetBitSet, c1.getMemo().getRefreshVersion());
@@ -282,7 +284,7 @@ class StructInfoMapTest extends SqlTestBase {
System.out.println(structInfo.getOriginalPlan().treeString());
BitSet bitSet = new BitSet();
for (CatalogRelation relation : structInfo.getRelations()) {
- bitSet.set(relation.getRelationId().asInt());
+ bitSet.set(c1.getStatementContext().getTableId(relation.getTable()).asInt());
}
Assertions.assertEquals(bitSet, mvMap);
dropMvByNereids("drop materialized view mv1");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index 24c421532f3..aa2d6d219dc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -254,6 +254,11 @@ public class NereidsParserTest extends ParserTestBase {
logicalJoin = (LogicalJoin) logicalPlan.child(0);
Assertions.assertEquals(JoinType.INNER_JOIN, logicalJoin.getJoinType());
+ String asofInnerJoin = "SELECT t1.a FROM t1 ASOF INNER JOIN t2 MATCH_CONDITION(t1.dt < t2.dt) ON t1.id = t2.id;";
+ logicalPlan = (LogicalPlan) nereidsParser.parseSingle(asofInnerJoin).child(0);
+ logicalJoin = (LogicalJoin) logicalPlan.child(0);
+ Assertions.assertEquals(JoinType.ASOF_LEFT_INNER_JOIN, logicalJoin.getJoinType());
+
String leftJoin1 = "SELECT t1.a FROM t1 LEFT JOIN t2 ON t1.id = t2.id;";
logicalPlan = (LogicalPlan) nereidsParser.parseSingle(leftJoin1).child(0);
logicalJoin = (LogicalJoin) logicalPlan.child(0);
@@ -264,6 +269,11 @@ public class NereidsParserTest extends ParserTestBase {
logicalJoin = (LogicalJoin) logicalPlan.child(0);
Assertions.assertEquals(JoinType.LEFT_OUTER_JOIN, logicalJoin.getJoinType());
+ String asofLeftJoin = "SELECT t1.a FROM t1 ASOF JOIN t2 MATCH_CONDITION(t1.dt < t2.dt) ON t1.id = t2.id;";
+ logicalPlan = (LogicalPlan) nereidsParser.parseSingle(asofLeftJoin).child(0);
+ logicalJoin = (LogicalJoin) logicalPlan.child(0);
+ Assertions.assertEquals(JoinType.ASOF_LEFT_OUTER_JOIN, logicalJoin.getJoinType());
+
String rightJoin1 = "SELECT t1.a FROM t1 RIGHT JOIN t2 ON t1.id = t2.id;";
logicalPlan = (LogicalPlan) nereidsParser.parseSingle(rightJoin1).child(0);
logicalJoin = (LogicalJoin) logicalPlan.child(0);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java
index a68479421b7..a0e2639771a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java
@@ -125,7 +125,13 @@ class ChildOutputPropertyDeriverTest {
@Test
void testInnerJoin() {
- PhysicalHashJoin<GroupPlan, GroupPlan> join = new PhysicalHashJoin<>(JoinType.INNER_JOIN,
+ testInnerJoinHelper(JoinType.INNER_JOIN);
+ testInnerJoinHelper(JoinType.ASOF_LEFT_INNER_JOIN);
+ testInnerJoinHelper(JoinType.ASOF_RIGHT_INNER_JOIN);
+ }
+
+ private void testInnerJoinHelper(JoinType joinType) {
+ PhysicalHashJoin<GroupPlan, GroupPlan> join = new PhysicalHashJoin<>(joinType,
ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE), Optional.empty(), logicalProperties,
groupPlan, groupPlan);
GroupExpression groupExpression = new GroupExpression(join);
@@ -203,7 +209,12 @@ class ChildOutputPropertyDeriverTest {
@Test
void testLeftOuterJoin() {
- PhysicalHashJoin<GroupPlan, GroupPlan> join = new PhysicalHashJoin<>(JoinType.LEFT_OUTER_JOIN,
+ testLeftOuterJoinHelper(JoinType.LEFT_OUTER_JOIN);
+ testLeftOuterJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void testLeftOuterJoinHelper(JoinType joinType) {
+ PhysicalHashJoin<GroupPlan, GroupPlan> join = new PhysicalHashJoin<>(joinType,
ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE), Optional.empty(), logicalProperties,
groupPlan, groupPlan);
GroupExpression groupExpression = new GroupExpression(join);
@@ -450,7 +461,12 @@ class ChildOutputPropertyDeriverTest {
@Test
void testRightOuterJoin() {
- PhysicalHashJoin<GroupPlan, GroupPlan> join = new PhysicalHashJoin<>(JoinType.RIGHT_OUTER_JOIN,
+ testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN);
+ testRightOuterJoinHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testRightOuterJoinHelper(JoinType joinType) {
+ PhysicalHashJoin<GroupPlan, GroupPlan> join = new PhysicalHashJoin<>(joinType,
ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, new DistributeHint(DistributeType.NONE), Optional.empty(), logicalProperties,
groupPlan, groupPlan);
GroupExpression groupExpression = new GroupExpression(join);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java
index 50cad2811ac..d2f9dffce37 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindExpressionTest.java
@@ -49,7 +49,7 @@ class BindExpressionTest extends TestWithFeService implements GeneratedPlanPatte
sql = String.format("select * from t1 %s t2",
joinType.toString().replace("_", " "));
}
- if (joinType.isNullAwareLeftAntiJoin()) {
+ if (joinType.isNullAwareLeftAntiJoin() || joinType.isAsofJoin()) {
continue;
}
PlanChecker.from(connectContext)
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java
index 63fa700f23c..abb41a6138c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindSlotReferenceTest.java
@@ -47,6 +47,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Optional;
class BindSlotReferenceTest implements MemoPatternMatchSupported {
@@ -161,12 +162,12 @@ class BindSlotReferenceTest implements MemoPatternMatchSupported {
DistributeHint hint = new DistributeHint(DistributeType.NONE);
LogicalUsingJoin<LogicalSubQueryAlias<LogicalOlapScan>, LogicalSubQueryAlias<LogicalOlapScan>>
using1 = new LogicalUsingJoin<>(JoinType.LEFT_OUTER_JOIN, sub1,
- sub2, ImmutableList.of(new UnboundSlot("id")), hint);
+ sub2, ImmutableList.of(new UnboundSlot("id")), Optional.empty(), hint);
LogicalUsingJoin<LogicalUsingJoin<LogicalSubQueryAlias<LogicalOlapScan>, LogicalSubQueryAlias<LogicalOlapScan>>,
LogicalSubQueryAlias<LogicalOlapScan>> using2 = new LogicalUsingJoin<>(
JoinType.LEFT_OUTER_JOIN, using1, sub3,
- ImmutableList.of(new UnboundSlot("id")), hint);
+ ImmutableList.of(new UnboundSlot("id")), Optional.empty(), hint);
PlanChecker.from(MemoTestUtils.createConnectContext())
.analyze(using2)
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java
index f0774d9b5e6..b41f40a265b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/JoinCommuteTest.java
@@ -43,11 +43,15 @@ import java.util.stream.Collectors;
public class JoinCommuteTest implements MemoPatternMatchSupported {
@Test
void testInnerJoinCommute() {
+ testInnerJoinCommuteHelper(JoinType.INNER_JOIN);
+ }
+
+ private void testInnerJoinCommuteHelper(JoinType joinType) {
LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0);
LogicalPlan join = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), join)
@@ -65,11 +69,16 @@ public class JoinCommuteTest implements MemoPatternMatchSupported {
@Test
void testParallelJoinCommute() {
+ testParallelJoinCommuteHelper(JoinType.LEFT_OUTER_JOIN);
+ testParallelJoinCommuteHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void testParallelJoinCommuteHelper(JoinType joinType) {
LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0);
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.build();
join = join.withJoinConjuncts(
ImmutableList.of(),
@@ -83,11 +92,15 @@ public class JoinCommuteTest implements MemoPatternMatchSupported {
@Test
void testJoinConjunctNullableWhenCommute() {
+ testJoinConjunctNullableWhenCommuteHelper(JoinType.LEFT_OUTER_JOIN, JoinType.RIGHT_OUTER_JOIN);
+ }
+
+ private void testJoinConjunctNullableWhenCommuteHelper(JoinType joinType1, JoinType joinType2) {
LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0);
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType1, Pair.of(0, 0))
.build();
join = join.withJoinConjuncts(
ImmutableList.of(new EqualTo(scan1.getOutput().get(0).withNullable(true),
@@ -101,7 +114,7 @@ public class JoinCommuteTest implements MemoPatternMatchSupported {
.getAllPlan()
.stream()
.filter(plan -> plan instanceof LogicalJoin
- && ((LogicalJoin<?, ?>) plan).getJoinType() == JoinType.RIGHT_OUTER_JOIN)
+ && ((LogicalJoin<?, ?>) plan).getJoinType() == joinType2)
.collect(Collectors.toList());
Assertions.assertEquals(1, allPlan.size());
// the input slot of join conjuncts should be nullable false after commute
@@ -123,4 +136,46 @@ public class JoinCommuteTest implements MemoPatternMatchSupported {
});
}
}
+
+ @Test
+ void testAsofJoinOtherConjunctsNotCommute() {
+ testAsofJoinOtherConjunctsNotCommuteHelper(JoinType.ASOF_LEFT_INNER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN);
+ testAsofJoinOtherConjunctsNotCommuteHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void testAsofJoinOtherConjunctsNotCommuteHelper(JoinType joinType, JoinType expectedSwap) {
+ LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0);
+
+ LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0))
+ .build();
+
+ // represent ASOF match_condition as otherConjuncts (e.g. t1.ts > t2.ts)
+ join = join.withJoinConjuncts(
+ join.getHashJoinConjuncts(),
+ ImmutableList.of(new GreaterThan(scan1.getOutput().get(0), scan2.getOutput().get(0))),
+ join.getJoinReorderContext());
+
+ List<Plan> allPlan = PlanChecker.from(MemoTestUtils.createConnectContext(), join)
+ .applyExploration(JoinCommute.BUSHY.build())
+ .getAllPlan()
+ .stream()
+ .filter(plan -> plan instanceof LogicalJoin
+ && ((LogicalJoin<?, ?>) plan).getJoinType() == expectedSwap)
+ .collect(Collectors.toList());
+ Assertions.assertEquals(1, allPlan.size());
+
+ LogicalJoin<?, ?> newJoin = (LogicalJoin<?, ?>) allPlan.get(0);
+ // other conjuncts should be commuted: left becomes scan2, right becomes scan1
+ Assertions.assertTrue(newJoin.getJoinType().isAsofLeftJoin());
+ Assertions.assertEquals(1, newJoin.getOtherJoinConjuncts().size());
+ Expression expr = newJoin.getOtherJoinConjuncts().get(0);
+ Assertions.assertTrue(expr instanceof GreaterThan);
+ GreaterThan gt = (GreaterThan) expr;
+ Assertions.assertTrue(gt.left() instanceof SlotReference);
+ Assertions.assertTrue(gt.right() instanceof SlotReference);
+ Assertions.assertEquals(scan1.getOutput().get(0), gt.left());
+ Assertions.assertEquals(scan2.getOutput().get(0), gt.right());
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java
index e5ba37b3bd2..4c1370da1c0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/join/PushDownProjectThroughInnerOuterJoinTest.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
@@ -42,8 +43,16 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
private final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0);
private final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0);
+ private ConnectContext connectContext = MemoTestUtils.createConnectContext();
+
@Test
public void pushBothSide() {
+ pushBothSideHelper(JoinType.INNER_JOIN);
+ pushBothSideHelper(JoinType.ASOF_LEFT_INNER_JOIN);
+ pushBothSideHelper(JoinType.ASOF_RIGHT_INNER_JOIN);
+ }
+
+ private void pushBothSideHelper(JoinType joinType) {
// project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name
List<NamedExpression> projectExprs = ImmutableList.of(
new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"),
@@ -53,12 +62,12 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
);
// complex projection contain ti.id, which isn't in Join Condition
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(1, 1))
+ .join(scan2, joinType, Pair.of(1, 1))
.projectExprs(projectExprs)
- .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1))
+ .join(scan3, joinType, Pair.of(1, 1))
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
@@ -75,6 +84,15 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
@Test
public void pushRightSide() {
+ pushRightSideHelper(JoinType.LEFT_OUTER_JOIN, JoinType.INNER_JOIN);
+ pushRightSideHelper(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN);
+ pushRightSideHelper(JoinType.LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN);
+ pushRightSideHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.INNER_JOIN);
+ pushRightSideHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN);
+ pushRightSideHelper(JoinType.ASOF_LEFT_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN);
+ }
+
+ private void pushRightSideHelper(JoinType bottom, JoinType top) {
// project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name
List<NamedExpression> projectExprs = ImmutableList.of(
new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"),
@@ -83,22 +101,22 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
);
// complex projection contain ti.id, which isn't in Join Condition
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(1, 1))
+ .join(scan2, bottom, Pair.of(1, 1))
.projectExprs(projectExprs)
- .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1))
+ .join(scan3, top, Pair.of(1, 1))
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
logicalJoin(
logicalProject(
- logicalJoin(
- logicalProject().when(project -> project.getProjects().size() == 2),
- logicalOlapScan()
- )
+ logicalJoin(
+ logicalProject().when(project -> project.getProjects().size() == 2),
+ logicalOlapScan()
+ )
),
logicalOlapScan()
)
@@ -107,6 +125,12 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
@Test
public void pushNoSide() {
+ pushNoSideHelper(JoinType.FULL_OUTER_JOIN, JoinType.INNER_JOIN);
+ pushNoSideHelper(JoinType.FULL_OUTER_JOIN, JoinType.ASOF_LEFT_INNER_JOIN);
+ pushNoSideHelper(JoinType.FULL_OUTER_JOIN, JoinType.ASOF_RIGHT_INNER_JOIN);
+ }
+
+ private void pushNoSideHelper(JoinType bottom, JoinType top) {
// project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name
List<NamedExpression> projectExprs = ImmutableList.of(
new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"),
@@ -115,12 +139,12 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
);
// complex projection contain ti.id, which isn't in Join Condition
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(1, 1))
+ .join(scan2, bottom, Pair.of(1, 1))
.projectExprs(projectExprs)
- .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1))
+ .join(scan3, top, Pair.of(1, 1))
.build();
- int plansNumber = PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ int plansNumber = PlanChecker.from(connectContext, plan)
.applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.plansNumber();
Assertions.assertEquals(1, plansNumber);
@@ -128,6 +152,12 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
@Test
public void pushdownProjectInCondition() {
+ pushdownProjectInConditionHelper(JoinType.INNER_JOIN);
+ pushdownProjectInConditionHelper(JoinType.ASOF_LEFT_INNER_JOIN);
+ pushdownProjectInConditionHelper(JoinType.ASOF_RIGHT_INNER_JOIN);
+ }
+
+ private void pushdownProjectInConditionHelper(JoinType joinType) {
// project (t1.id + 1) as alias, t1.name, (t2.id + 1) as alias, t2.name
List<NamedExpression> projectExprs = ImmutableList.of(
new Alias(new Add(scan1.getOutput().get(0), Literal.of(1)), "alias"),
@@ -137,12 +167,12 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
);
// complex projection contain ti.id, which is in Join Condition
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.projectExprs(projectExprs)
- .join(scan3, JoinType.INNER_JOIN, Pair.of(1, 1))
+ .join(scan3, joinType, Pair.of(1, 1))
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
@@ -161,6 +191,12 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
@Test
void pushComplexProject() {
+ pushComplexProjectHelper(JoinType.INNER_JOIN);
+ pushComplexProjectHelper(JoinType.ASOF_LEFT_INNER_JOIN);
+ pushComplexProjectHelper(JoinType.ASOF_RIGHT_INNER_JOIN);
+ }
+
+ private void pushComplexProjectHelper(JoinType joinType) {
// project (t1.id + t1.name) as complex1, (t2.id + t2.name) as complex2
List<NamedExpression> projectExprs = ImmutableList.of(
new Alias(new Add(scan1.getOutput().get(0), scan1.getOutput().get(1)), "complex1"),
@@ -168,49 +204,55 @@ class PushDownProjectThroughInnerOuterJoinTest implements MemoPatternMatchSuppor
);
// complex projection contain ti.id, which is in Join Condition
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.projectExprs(projectExprs)
- .join(scan3, JoinType.INNER_JOIN, Pair.of(0, 0))
+ .join(scan3, joinType, Pair.of(0, 0))
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.printlnOrigin()
.printlnExploration()
.matchesExploration(
- logicalJoin(
- logicalProject(
- logicalJoin(
- logicalProject()
- .when(project ->
- project.getProjects().get(0).toSql().equals("(id + name) AS `complex1`")
- && project.getProjects().get(1).toSql().equals("id")),
- logicalProject()
- .when(project ->
- project.getProjects().get(0).toSql().equals("(id + name) AS `complex2`")
- && project.getProjects().get(1).toSql().equals("id"))
- )
- ).when(project -> project.getProjects().get(0).toSql().equals("complex1")
- && project.getProjects().get(1).toSql().equals("complex2")),
- logicalOlapScan()
- )
+ logicalJoin(
+ logicalProject(
+ logicalJoin(
+ logicalProject()
+ .when(project ->
+ project.getProjects().get(0).toSql().equals("(id + name) AS `complex1`")
+ && project.getProjects().get(1).toSql().equals("id")),
+ logicalProject()
+ .when(project ->
+ project.getProjects().get(0).toSql().equals("(id + name) AS `complex2`")
+ && project.getProjects().get(1).toSql().equals("id"))
+ )
+ ).when(project -> project.getProjects().get(0).toSql().equals("complex1")
+ && project.getProjects().get(1).toSql().equals("complex2")),
+ logicalOlapScan()
+ )
);
}
@Test
void rejectHyperEdgeProject() {
+ rejectHyperEdgeProjectHelper(JoinType.INNER_JOIN);
+ rejectHyperEdgeProjectHelper(JoinType.ASOF_LEFT_INNER_JOIN);
+ rejectHyperEdgeProjectHelper(JoinType.ASOF_RIGHT_INNER_JOIN);
+ }
+
+ private void rejectHyperEdgeProjectHelper(JoinType joinType) {
// project (t1.id + t2.id) as alias
List<NamedExpression> projectExprs = ImmutableList.of(
new Alias(new Add(scan1.getOutput().get(0), scan2.getOutput().get(0)), "alias")
);
// complex projection contain ti.id, which is in Join Condition
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.projectExprs(projectExprs)
- .join(scan3, JoinType.INNER_JOIN, Pair.of(0, 0))
+ .join(scan3, joinType, Pair.of(0, 0))
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyExploration(PushDownProjectThroughInnerOuterJoin.INSTANCE.buildRules())
.checkMemo(memo -> Assertions.assertEquals(1, memo.getRoot().getLogicalExpressions().size()));
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java
index eac9e0ecc25..30b8c9166fb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagationTest.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.MatchAny;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -431,6 +432,33 @@ class ConstantPropagationTest {
Assertions.assertEquals(expectOtherConjuncts1, rewrittenJoin1.getOtherJoinConjuncts());
}
+ @Test
+ void testAsofJoinDisableConstantPropagation() {
+ Set<Expression> conjunctions1 = ImmutableSet.of(
+ new EqualTo(studentId, new IntegerLiteral(1))
+ );
+ Set<Expression> conjunctions2 = ImmutableSet.of(
+ new EqualTo(scoreCid, new IntegerLiteral(2))
+ );
+ LogicalFilter left = new LogicalFilter<>(conjunctions1, student);
+ LogicalFilter right = new LogicalFilter<>(conjunctions2, score);
+
+ List<Expression> hashConjuncts = ImmutableList.of(
+ new EqualTo(studentId, scoreSid)
+ );
+ List<Expression> otherConjuncts = ImmutableList.of(
+ new GreaterThan(studentId, studentAge)
+ );
+
+ LogicalJoin asofJoin = new LogicalJoin<>(JoinType.ASOF_LEFT_INNER_JOIN, hashConjuncts, otherConjuncts, left, right, null);
+ LogicalJoin rewrittenAsofJoin = (LogicalJoin) executor.rewriteRoot(asofJoin, jobContext);
+
+ // For ASOF join, constant propagation should be disabled and no rewrite should happen.
+ Assertions.assertEquals(JoinType.ASOF_LEFT_INNER_JOIN, rewrittenAsofJoin.getJoinType());
+ Assertions.assertEquals(hashConjuncts, rewrittenAsofJoin.getHashJoinConjuncts());
+ Assertions.assertEquals(otherConjuncts, rewrittenAsofJoin.getOtherJoinConjuncts());
+ }
+
private void assertRewrite(String expression, String expected) {
assertRewrite(expression, expected, true);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java
index 6ecb50e53ff..229459d7824 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java
@@ -81,6 +81,26 @@ class ConvertOuterJoinToAntiJoinTest implements MemoPatternMatchSupported {
.matches(logicalJoin().when(join -> join.getJoinType().isRightAntiJoin()));
}
+ @Test
+ void testNoEliminateAsofWithProject() {
+ testNoEliminateAsofWithProjectHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ testNoEliminateAsofWithProjectHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testNoEliminateAsofWithProjectHelper(JoinType joinType) {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
+ .filter(new IsNull(scan2.getOutput().get(0)))
+ .projectExprs(ImmutableList.copyOf(scan1.getOutput()))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new InferFilterNotNull())
+ .applyCustom(new ConvertOuterJoinToAntiJoin())
+ .printlnTree()
+ .matches(logicalJoin().when(join -> join.getJoinType() == joinType));
+ }
+
@Test
void testEliminateLeftWithLeftPredicate() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
@@ -99,6 +119,29 @@ class ConvertOuterJoinToAntiJoinTest implements MemoPatternMatchSupported {
.matches(logicalJoin().when(join -> join.getJoinType().isLeftAntiJoin()));
}
+ @Test
+ void testNoEliminateAsofWithLeftPredicate() {
+ testNoEliminateAsofWithLeftPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ testNoEliminateAsofWithLeftPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testNoEliminateAsofWithLeftPredicateHelper(JoinType joinType) {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
+ .filter(Sets.newHashSet(
+ new IsNull(scan2.getOutput().get(0)),
+ new EqualTo(scan1.getOutput().get(0), new IntegerLiteral(1)))
+ )
+ .projectExprs(ImmutableList.copyOf(scan1.getOutput()))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new InferFilterNotNull())
+ .applyCustom(new ConvertOuterJoinToAntiJoin())
+ .printlnTree()
+ .matches(logicalJoin().when(join -> join.getJoinType() == joinType));
+ }
+
@Test
void testEliminateLeftWithRightPredicate() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
@@ -117,10 +160,39 @@ class ConvertOuterJoinToAntiJoinTest implements MemoPatternMatchSupported {
.matches(logicalJoin().when(join -> join.getJoinType().isLeftAntiJoin()));
}
+ @Test
+ void testNoEliminateAsofWithRightPredicate() {
+ testNoEliminateAsofWithRightPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ testNoEliminateAsofWithRightPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testNoEliminateAsofWithRightPredicateHelper(JoinType joinType) {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
+ .filter(Sets.newHashSet(
+ new IsNull(scan2.getOutput().get(0)),
+ new EqualTo(scan2.getOutput().get(0), new IntegerLiteral(1)))
+ )
+ .projectExprs(ImmutableList.copyOf(scan1.getOutput()))
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new InferFilterNotNull())
+ .applyCustom(new ConvertOuterJoinToAntiJoin())
+ .printlnTree()
+ .matches(logicalJoin().when(join -> join.getJoinType() == joinType));
+ }
+
@Test
void testEliminateLeftWithOrPredicate() {
+ testEliminateLeftWithOrPredicateHelper(JoinType.LEFT_OUTER_JOIN);
+ testEliminateLeftWithOrPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ testEliminateLeftWithOrPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testEliminateLeftWithOrPredicateHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
.filter(Sets.newHashSet(
new IsNull(scan1.getOutput().get(0)),
new Or(new IsNull(scan1.getOutput().get(0)), new IsNull(scan2.getOutput().get(0))))
@@ -132,13 +204,19 @@ class ConvertOuterJoinToAntiJoinTest implements MemoPatternMatchSupported {
.applyTopDown(new InferFilterNotNull())
.applyCustom(new ConvertOuterJoinToAntiJoin())
.printlnTree()
- .matches(logicalJoin().when(join -> join.getJoinType().isLeftOuterJoin()));
+ .matches(logicalJoin().when(join -> join.getJoinType() == joinType));
}
@Test
void testEliminateLeftWithAndPredicate() {
+ testEliminateLeftWithAndPredicateHelper(JoinType.LEFT_OUTER_JOIN);
+ testEliminateLeftWithAndPredicateHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ testEliminateLeftWithAndPredicateHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testEliminateLeftWithAndPredicateHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
.filter(Sets.newHashSet(
new IsNull(scan1.getOutput().get(0)),
new EqualTo(scan1.getOutput().get(0), new IntegerLiteral(1)),
@@ -151,6 +229,6 @@ class ConvertOuterJoinToAntiJoinTest implements MemoPatternMatchSupported {
.applyTopDown(new InferFilterNotNull())
.applyCustom(new ConvertOuterJoinToAntiJoin())
.printlnTree()
- .matches(logicalJoin().when(join -> join.getJoinType().isLeftOuterJoin()));
+ .matches(logicalJoin().when(join -> join.getJoinType() == joinType));
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java
index f9addf9fae3..977c216c4e7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateConstHashJoinConditionTest.java
@@ -55,4 +55,20 @@ public class EliminateConstHashJoinConditionTest extends SqlTestBase {
);
}
+ @Test
+ void testNotEliminateAsofJoin() {
+ CascadesContext c1 = createCascadesContext(
+ "select * from T1 asof inner join T2 match_condition(cast(T1.score as datetime) "
+ + "> cast(T2.score as datetime)) on T1.id = T2.id where T1.id = 1",
+ connectContext
+ );
+ PlanChecker.from(c1)
+ .analyze()
+ .rewrite()
+ .matches(
+ logicalJoin().when(join ->
+ !join.getHashJoinConjuncts().isEmpty())
+ );
+ }
+
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
index 7622ae601ca..0d54ad4f1e5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
@@ -290,4 +290,25 @@ class EliminateJoinByFkTest extends TestWithFeService implements MemoPatternMatc
oneRowRelation);
Assertions.assertEquals(expectFilter, result2.first);
}
+
+ @Test
+ void testAsofJoinUnchange() {
+ String sql = "select pri.id1 from pri asof inner join foreign_not_null "
+ + "match_condition(cast(pri.id1 as datetime) > cast(foreign_not_null.id2 as datetime)) "
+ + "on pri.id1 = foreign_not_null.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .anyMatches(logicalJoin())
+ .printlnTree();
+
+ sql = "select foreign_not_null.id2 from pri asof join foreign_not_null "
+ + "match_condition(cast(pri.id1 as datetime) > cast(foreign_not_null.id2 as datetime)) "
+ + "on pri.id1 = foreign_not_null.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .anyMatches(logicalJoin())
+ .printlnTree();
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java
index 8da050a0e47..88c9ecc339e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java
@@ -32,13 +32,15 @@ class EliminateJoinByUniqueTest extends TestWithFeService implements MemoPattern
createTables(
"CREATE TABLE IF NOT EXISTS t1 (\n"
+ " id1 int not null,\n"
- + " id_null int\n"
+ + " id_null int,\n"
+ + " dt1 date not null\n"
+ ")\n"
+ "DUPLICATE KEY(id1)\n"
+ "DISTRIBUTED BY HASH(id1) BUCKETS 10\n"
+ "PROPERTIES (\"replication_num\" = \"1\")\n",
"CREATE TABLE IF NOT EXISTS t2 (\n"
- + " id2 int not null\n"
+ + " id2 int not null,\n"
+ + " dt2 date not null\n"
+ ")\n"
+ "DUPLICATE KEY(id2)\n"
+ "DISTRIBUTED BY HASH(id2) BUCKETS 10\n"
@@ -56,6 +58,12 @@ class EliminateJoinByUniqueTest extends TestWithFeService implements MemoPattern
.rewrite()
.nonMatch(logicalJoin())
.printlnTree();
+ sql = "select t1.id1 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id1 = t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .anyMatches(logicalJoin())
+ .printlnTree();
sql = "select t2.id2 from t1 left outer join t2 on t1.id1 = t2.id2";
PlanChecker.from(connectContext)
@@ -63,6 +71,12 @@ class EliminateJoinByUniqueTest extends TestWithFeService implements MemoPattern
.rewrite()
.matches(logicalJoin())
.printlnTree();
+ sql = "select t2.id2 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id1 = t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(logicalJoin())
+ .printlnTree();
}
@Test
@@ -73,6 +87,12 @@ class EliminateJoinByUniqueTest extends TestWithFeService implements MemoPattern
.rewrite()
.nonMatch(logicalJoin())
.printlnTree();
+ sql = "select t1.id1 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id_null = t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .anyMatches(logicalJoin())
+ .printlnTree();
sql = "select t2.id2 from t1 left outer join t2 on t1.id_null = t2.id2";
PlanChecker.from(connectContext)
@@ -80,5 +100,11 @@ class EliminateJoinByUniqueTest extends TestWithFeService implements MemoPattern
.rewrite()
.matches(logicalJoin())
.printlnTree();
+ sql = "select t2.id2 from t1 asof left join t2 MATCH_CONDITION(t1.dt1 > t2.dt2) on t1.id_null = t2.id2";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(logicalJoin())
+ .printlnTree();
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java
index f0034410163..850c31a14ab 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java
@@ -49,8 +49,13 @@ class EliminateOuterJoinTest implements MemoPatternMatchSupported {
@Test
void testEliminateLeft() {
+ testEliminateLeftHelper(JoinType.LEFT_OUTER_JOIN);
+ testEliminateLeftHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void testEliminateLeftHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
.filter(new GreaterThan(scan2.getOutput().get(0), Literal.of(1)))
.build();
@@ -61,15 +66,21 @@ class EliminateOuterJoinTest implements MemoPatternMatchSupported {
.printlnTree()
.matchesFromRoot(
logicalFilter(
- logicalJoin().when(join -> join.getJoinType().isInnerJoin())
- ).when(filter -> filter.getConjuncts().size() == 1)
+ logicalJoin().when(join -> join.getJoinType().isInnerJoin()
+ || join.getJoinType().isAsofInnerJoin()))
+ .when(filter -> filter.getConjuncts().size() == 1)
);
}
@Test
void testEliminateRight() {
+ testEliminateRightHelper(JoinType.RIGHT_OUTER_JOIN);
+ testEliminateRightHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testEliminateRightHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
+ .join(scan2, joinType, Pair.of(0, 0)) // t1.id = t2.id
.filter(new GreaterThan(scan1.getOutput().get(0), new IntegerLiteral(1)))
.build();
@@ -80,9 +91,11 @@ class EliminateOuterJoinTest implements MemoPatternMatchSupported {
.printlnTree()
.matchesFromRoot(
logicalFilter(
- logicalJoin().when(join -> join.getJoinType().isInnerJoin())
- ).when(filter -> filter.getConjuncts().size() == 1)
- .when(filter -> Objects.equals(filter.getConjuncts().iterator().next(), new GreaterThan(scan1.getOutput().get(0), new IntegerLiteral(1))))
+ logicalJoin().when(join -> join.getJoinType().isInnerJoin()
+ || join.getJoinType().isAsofInnerJoin()))
+ .when(filter -> filter.getConjuncts().size() == 1)
+ .when(filter -> Objects.equals(filter.getConjuncts().iterator().next(),
+ new GreaterThan(scan1.getOutput().get(0), new IntegerLiteral(1))))
);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java
index a131ba7f2dd..1b1bf8afcf2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java
@@ -52,6 +52,7 @@ class InferPredicatesTest extends TestWithFeService implements MemoPatternMatchS
createTable("create table test.student (\n"
+ "id int not null,\n"
+ "name varchar(128),\n"
+ + "dt date,\n"
+ "age int,sex int)\n"
+ "distributed by hash(id) buckets 10\n"
+ "properties('replication_num' = '1');");
@@ -59,6 +60,7 @@ class InferPredicatesTest extends TestWithFeService implements MemoPatternMatchS
createTable("create table test.score (\n"
+ "sid int not null, \n"
+ "cid int not null, \n"
+ + "dt date,\n"
+ "grade double)\n"
+ "distributed by hash(sid,cid) buckets 10\n"
+ "properties('replication_num' = '1');");
@@ -66,27 +68,28 @@ class InferPredicatesTest extends TestWithFeService implements MemoPatternMatchS
createTable("create table test.course (\n"
+ "id int not null, \n"
+ "name varchar(128), \n"
+ + "dt date,\n"
+ "teacher varchar(128))\n"
+ "distributed by hash(id) buckets 10\n"
+ "properties('replication_num' = '1');");
createTables("create table test.subquery1\n"
- + "(k1 bigint, k2 bigint)\n"
+ + "(k1 bigint, k2 bigint, dt date)\n"
+ "duplicate key(k1)\n"
+ "distributed by hash(k2) buckets 1\n"
+ "properties('replication_num' = '1');\n",
"create table test.subquery2\n"
- + "(k1 varchar(10), k2 bigint)\n"
+ + "(k1 varchar(10), k2 bigint, dt date)\n"
+ "partition by range(k2)\n"
+ "(partition p1 values less than(\"10\"))\n"
+ "distributed by hash(k2) buckets 1\n"
+ "properties('replication_num' = '1');",
"create table test.subquery3\n"
- + "(k1 int not null, k2 varchar(128), k3 bigint, v1 bigint, v2 bigint)\n"
+ + "(k1 int not null, k2 varchar(128), k3 bigint, v1 bigint, v2 bigint, dt date)\n"
+ "distributed by hash(k2) buckets 1\n"
+ "properties('replication_num' = '1');",
"create table test.subquery4\n"
- + "(k1 bigint, k2 bigint)\n"
+ + "(k1 bigint, k2 bigint, dt date)\n"
+ "duplicate key(k1)\n"
+ "distributed by hash(k2) buckets 1\n"
+ "properties('replication_num' = '1');");
@@ -859,4 +862,49 @@ class InferPredicatesTest extends TestWithFeService implements MemoPatternMatchS
Assertions.assertTrue(allPredicates.contains(leftPredicate));
Assertions.assertFalse(allPredicates.contains(rightPredicate));
}
+
+ @Test
+ void inferPredicatesLeftAsofLeft() {
+ String sql = "select * from student asof left join score match_condition(student.dt > score.dt) on student.id = score.sid where student.id > 1";
+
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matches(
+ logicalJoin(
+ logicalFilter(
+ logicalOlapScan()
+ ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+ & filter.getPredicate().toSql().contains("id > 1")),
+ logicalFilter(
+ logicalOlapScan()
+ ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+ & filter.getPredicate().toSql().contains("sid > 1"))
+ )
+ );
+ }
+
+ @Test
+ void inferPredicatesLeftAsofInner() {
+ // convert left join to inner join
+ String sql = "select * from student asof left join score match_condition(student.dt > score.dt) on student.id = score.sid where score.sid > 1";
+
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .printlnTree()
+ .matches(
+ logicalJoin(
+ logicalFilter(
+ logicalOlapScan()
+ ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+ & filter.getPredicate().toSql().contains("id > 1")),
+ logicalFilter(
+ logicalOlapScan()
+ ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+ & filter.getPredicate().toSql().contains("sid > 1"))
+ )
+ );
+ }
+
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java
index 343619cbd8b..fb8be4ea1f6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoinTest.java
@@ -31,6 +31,7 @@ import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -49,6 +50,8 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
private LogicalPlan rStudent;
private LogicalPlan rScore;
+ private ConnectContext connectContext;
+
/**
* ut before.
*/
@@ -57,16 +60,20 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
rStudent = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student,
ImmutableList.of(""));
rScore = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.score, ImmutableList.of(""));
+ connectContext = MemoTestUtils.createConnectContext();
}
@Test
public void oneSide() {
testLeft(JoinType.CROSS_JOIN);
testLeft(JoinType.INNER_JOIN);
+ testLeft(JoinType.ASOF_LEFT_INNER_JOIN);
testLeft(JoinType.LEFT_OUTER_JOIN);
+ testLeft(JoinType.ASOF_LEFT_OUTER_JOIN);
testLeft(JoinType.LEFT_SEMI_JOIN);
testLeft(JoinType.LEFT_ANTI_JOIN);
testRight(JoinType.RIGHT_OUTER_JOIN);
+ testRight(JoinType.ASOF_RIGHT_OUTER_JOIN);
testRight(JoinType.RIGHT_SEMI_JOIN);
testRight(JoinType.RIGHT_ANTI_JOIN);
}
@@ -81,7 +88,7 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
.filter(whereCondition)
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyTopDown(PushDownFilterThroughJoin.INSTANCE)
.matchesFromRoot(
logicalJoin(
@@ -102,7 +109,7 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
.filter(whereCondition)
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyTopDown(PushDownFilterThroughJoin.INSTANCE)
.matchesFromRoot(
logicalJoin(
@@ -133,7 +140,7 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
.build();
if (joinType.isInnerJoin()) {
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyTopDown(PushDownFilterThroughJoin.INSTANCE)
.printlnTree()
.matchesFromRoot(
@@ -146,7 +153,7 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
);
}
if (joinType.isCrossJoin()) {
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyTopDown(PushDownFilterThroughJoin.INSTANCE)
.printlnTree()
.matchesFromRoot(
@@ -165,9 +172,11 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
@Test
public void bothSideToOneSide() {
bothSideToLeft(JoinType.LEFT_OUTER_JOIN);
+ bothSideToLeft(JoinType.ASOF_LEFT_OUTER_JOIN);
bothSideToLeft(JoinType.LEFT_ANTI_JOIN);
bothSideToLeft(JoinType.LEFT_SEMI_JOIN);
bothSideToRight(JoinType.RIGHT_OUTER_JOIN);
+ bothSideToRight(JoinType.ASOF_RIGHT_OUTER_JOIN);
bothSideToRight(JoinType.RIGHT_ANTI_JOIN);
bothSideToRight(JoinType.RIGHT_SEMI_JOIN);
}
@@ -182,7 +191,7 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
.filter(whereCondition)
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyTopDown(PushDownFilterThroughJoin.INSTANCE)
.matchesFromRoot(
logicalFilter(
@@ -205,7 +214,7 @@ public class PushDownFilterThroughJoinTest implements MemoPatternMatchSupported
.filter(whereCondition)
.build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ PlanChecker.from(connectContext, plan)
.applyTopDown(PushDownFilterThroughJoin.INSTANCE)
.matchesFromRoot(
logicalFilter(
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java
index 05642f084d5..20b20f180d0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownJoinOtherConditionTest.java
@@ -64,17 +64,43 @@ class PushDownJoinOtherConditionTest implements MemoPatternMatchSupported {
@Test
void oneSide() {
- oneSide(JoinType.CROSS_JOIN, false);
- oneSide(JoinType.INNER_JOIN, false);
- oneSide(JoinType.LEFT_OUTER_JOIN, true);
- oneSide(JoinType.LEFT_SEMI_JOIN, true);
- oneSide(JoinType.LEFT_ANTI_JOIN, true);
- oneSide(JoinType.RIGHT_OUTER_JOIN, false);
- oneSide(JoinType.RIGHT_SEMI_JOIN, false);
- oneSide(JoinType.RIGHT_ANTI_JOIN, false);
+ oneSidePush(JoinType.CROSS_JOIN, false);
+ oneSidePush(JoinType.INNER_JOIN, false);
+ oneSidePush(JoinType.LEFT_OUTER_JOIN, true);
+ oneSidePush(JoinType.LEFT_SEMI_JOIN, true);
+ oneSidePush(JoinType.LEFT_ANTI_JOIN, true);
+ oneSidePush(JoinType.RIGHT_OUTER_JOIN, false);
+ oneSidePush(JoinType.RIGHT_SEMI_JOIN, false);
+ oneSidePush(JoinType.RIGHT_ANTI_JOIN, false);
+ oneSideNoPush(JoinType.ASOF_LEFT_INNER_JOIN);
+ oneSideNoPush(JoinType.ASOF_RIGHT_INNER_JOIN);
+ oneSideNoPush(JoinType.ASOF_LEFT_OUTER_JOIN);
+ oneSideNoPush(JoinType.ASOF_RIGHT_OUTER_JOIN);
}
- private void oneSide(JoinType joinType, boolean testRight) {
+ private void oneSideNoPush(JoinType joinType) {
+ Expression pushSide1 = new GreaterThan(rStudentSlots.get(1), Literal.of(18));
+ Expression pushSide2 = new GreaterThan(rStudentSlots.get(1), Literal.of(50));
+ List<Expression> condition = ImmutableList.of(pushSide1, pushSide2);
+
+ LogicalOlapScan left = rStudent;
+ LogicalOlapScan right = rScore;
+
+ LogicalPlan root = new LogicalPlanBuilder(left)
+ .join(right, joinType, ExpressionUtils.EMPTY_CONDITION, condition)
+ .project(Lists.newArrayList())
+ .build();
+
+ PlanChecker planChecker = PlanChecker.from(MemoTestUtils.createConnectContext(), root)
+ .applyTopDown(new PushDownJoinOtherCondition());
+
+ planChecker.matches(
+ logicalJoin(
+ logicalOlapScan(),
+ logicalOlapScan()).when(join -> join.getOtherJoinConjuncts().equals(condition)));
+ }
+
+ private void oneSidePush(JoinType joinType, boolean testRight) {
Expression pushSide1 = new GreaterThan(rStudentSlots.get(1), Literal.of(18));
Expression pushSide2 = new GreaterThan(rStudentSlots.get(1), Literal.of(50));
List<Expression> condition = ImmutableList.of(pushSide1, pushSide2);
@@ -112,13 +138,15 @@ class PushDownJoinOtherConditionTest implements MemoPatternMatchSupported {
@Test
void bothSideToBothSide() {
- bothSideToBothSide(JoinType.CROSS_JOIN);
- bothSideToBothSide(JoinType.INNER_JOIN);
- bothSideToBothSide(JoinType.LEFT_SEMI_JOIN);
- bothSideToBothSide(JoinType.RIGHT_SEMI_JOIN);
+ bothSidePush(JoinType.CROSS_JOIN);
+ bothSidePush(JoinType.INNER_JOIN);
+ bothSidePush(JoinType.LEFT_SEMI_JOIN);
+ bothSidePush(JoinType.RIGHT_SEMI_JOIN);
+ bothSideNoPush(JoinType.ASOF_LEFT_INNER_JOIN);
+ bothSideNoPush(JoinType.ASOF_RIGHT_INNER_JOIN);
}
- private void bothSideToBothSide(JoinType joinType) {
+ private void bothSidePush(JoinType joinType) {
Expression leftSide = new GreaterThan(rStudentSlots.get(1), Literal.of(18));
Expression rightSide = new GreaterThan(rScoreSlots.get(2), Literal.of(60));
@@ -138,15 +166,35 @@ class PushDownJoinOtherConditionTest implements MemoPatternMatchSupported {
));
}
+ private void bothSideNoPush(JoinType joinType) {
+ Expression leftSide = new GreaterThan(rStudentSlots.get(1), Literal.of(18));
+ Expression rightSide = new GreaterThan(rScoreSlots.get(2), Literal.of(60));
+ List<Expression> condition = ImmutableList.of(leftSide, rightSide);
+
+ LogicalPlan root = new LogicalPlanBuilder(rStudent)
+ .join(rScore, joinType, ExpressionUtils.EMPTY_CONDITION, condition)
+ .project(Lists.newArrayList())
+ .build();
+
+ PlanChecker.from(MemoTestUtils.createConnectContext(), root)
+ .applyTopDown(new PushDownJoinOtherCondition())
+ .matches(
+ logicalJoin(
+ logicalOlapScan(),
+ logicalOlapScan()));
+ }
+
@Test
void bothSideToOneSide() {
- bothSideToOneSide(JoinType.LEFT_OUTER_JOIN, true);
- bothSideToOneSide(JoinType.LEFT_ANTI_JOIN, true);
- bothSideToOneSide(JoinType.RIGHT_OUTER_JOIN, false);
- bothSideToOneSide(JoinType.RIGHT_ANTI_JOIN, false);
+ bothSideToOneSidePush(JoinType.LEFT_OUTER_JOIN, true);
+ bothSideToOneSidePush(JoinType.LEFT_ANTI_JOIN, true);
+ bothSideToOneSidePush(JoinType.RIGHT_OUTER_JOIN, false);
+ bothSideToOneSidePush(JoinType.RIGHT_ANTI_JOIN, false);
+ bothSideToOneSideNoPush(JoinType.ASOF_LEFT_OUTER_JOIN);
+ bothSideToOneSideNoPush(JoinType.ASOF_RIGHT_OUTER_JOIN);
}
- private void bothSideToOneSide(JoinType joinType, boolean testRight) {
+ private void bothSideToOneSidePush(JoinType joinType, boolean testRight) {
Expression pushSide = new GreaterThan(rStudentSlots.get(1), Literal.of(18));
Expression reserveSide = new GreaterThan(rScoreSlots.get(2), Literal.of(60));
List<Expression> condition = ImmutableList.of(pushSide, reserveSide);
@@ -180,4 +228,26 @@ class PushDownJoinOtherConditionTest implements MemoPatternMatchSupported {
));
}
}
+
+ private void bothSideToOneSideNoPush(JoinType joinType) {
+ Expression pushSide = new GreaterThan(rStudentSlots.get(1), Literal.of(18));
+ Expression reserveSide = new GreaterThan(rScoreSlots.get(2), Literal.of(60));
+ List<Expression> condition = ImmutableList.of(pushSide, reserveSide);
+
+ LogicalOlapScan left = rStudent;
+ LogicalOlapScan right = rScore;
+
+ LogicalPlan root = new LogicalPlanBuilder(left)
+ .join(right, joinType, ExpressionUtils.EMPTY_CONDITION, condition)
+ .project(Lists.newArrayList())
+ .build();
+
+ PlanChecker planChecker = PlanChecker.from(MemoTestUtils.createConnectContext(), root)
+ .applyTopDown(new PushDownJoinOtherCondition());
+
+ planChecker.matches(
+ logicalJoin(
+ logicalOlapScan(),
+ logicalOlapScan()));
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java
index bce9a2cd207..6eb92c79125 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownLimitDistinctThroughJoinTest.java
@@ -84,6 +84,20 @@ class PushDownLimitDistinctThroughJoinTest extends TestWithFeService implements
)
);
+ plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.ASOF_LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .distinct(ImmutableList.of(0, 1))
+ .limit(10)
+ .build();
+ PlanChecker.from(connectContext, plan)
+ .applyTopDown(new PushDownLimitDistinctThroughJoin())
+ .matches(
+ logicalJoin(
+ logicalLimit(logicalAggregate(logicalOlapScan())).when(l -> l.getLimit() == 10),
+ logicalOlapScan()
+ )
+ );
+
plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0))
.distinct(ImmutableList.of(2, 3))
@@ -98,6 +112,20 @@ class PushDownLimitDistinctThroughJoinTest extends TestWithFeService implements
)
);
+ plan = new LogicalPlanBuilder(scan1)
+ .join(scan2, JoinType.ASOF_RIGHT_OUTER_JOIN, Pair.of(0, 0))
+ .distinct(ImmutableList.of(2, 3))
+ .limit(10)
+ .build();
+ PlanChecker.from(connectContext, plan)
+ .applyTopDown(new PushDownLimitDistinctThroughJoin())
+ .matches(
+ logicalJoin(
+ logicalOlapScan(),
+ logicalLimit(logicalAggregate(logicalOlapScan())).when(l -> l.getLimit() == 10)
+ )
+ );
+
plan = new LogicalPlanBuilder(scan1)
.join(scan2, JoinType.CROSS_JOIN, Pair.of(0, 0))
.distinct(ImmutableList.of(0, 1))
@@ -143,8 +171,13 @@ class PushDownLimitDistinctThroughJoinTest extends TestWithFeService implements
@Test
void badCaseJoinType() {
+ badCaseJoinTypeHelper(JoinType.LEFT_OUTER_JOIN);
+ badCaseJoinTypeHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void badCaseJoinTypeHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.distinct(ImmutableList.of(2))
.limit(10)
.build();
@@ -155,9 +188,14 @@ class PushDownLimitDistinctThroughJoinTest extends TestWithFeService implements
@Test
void badCaseOutput() {
+ badCaseOutputHelper(JoinType.LEFT_OUTER_JOIN);
+ badCaseOutputHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void badCaseOutputHelper(JoinType joinType) {
// distinct agg don't output all group by columns of left child
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.distinct(ImmutableList.of(0))
.limit(10)
.build();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java
index 1e54444be72..6d2ffb3bbbe 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughJoinTest.java
@@ -76,8 +76,13 @@ class PushDownTopNThroughJoinTest extends TestWithFeService implements MemoPatte
@Test
void testJoin() {
+ testJoinHelper(JoinType.LEFT_OUTER_JOIN);
+ testJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void testJoinHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.topN(10, 0, ImmutableList.of(0))
.build();
PlanChecker.from(connectContext, plan)
@@ -94,10 +99,15 @@ class PushDownTopNThroughJoinTest extends TestWithFeService implements MemoPatte
@Test
void testProject1() {
+ testProject1Helper(JoinType.LEFT_OUTER_JOIN);
+ testProject1Helper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void testProject1Helper(JoinType joinType) {
List<NamedExpression> projectExpres = ImmutableList.of(scan1.getOutput().get(0),
new Cast(scan1.getOutput().get(1), VarcharType.SYSTEM_DEFAULT).alias("cast"));
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.projectExprs(projectExpres)
.topN(10, 0, ImmutableList.of(0))
.build();
@@ -178,11 +188,16 @@ class PushDownTopNThroughJoinTest extends TestWithFeService implements MemoPatte
@Test
void rejectTopNUseProjectComplexExpr() {
+ rejectTopNUseProjectComplexExprHelper(JoinType.LEFT_OUTER_JOIN);
+ rejectTopNUseProjectComplexExprHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ }
+
+ private void rejectTopNUseProjectComplexExprHelper(JoinType joinType) {
List<NamedExpression> projectExpres = ImmutableList.of(
(new Add(scan1.getOutput().get(0), scan1.getOutput().get(1))).alias("add")
);
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.projectExprs(projectExpres)
.topN(10, 0, ImmutableList.of(0))
.build();
@@ -198,8 +213,13 @@ class PushDownTopNThroughJoinTest extends TestWithFeService implements MemoPatte
@Test
void rejectWrongJoinType() {
+ rejectWrongJoinTypeHelper(JoinType.RIGHT_OUTER_JOIN);
+ rejectWrongJoinTypeHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void rejectWrongJoinTypeHelper(JoinType joinType) {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.topN(10, 0, ImmutableList.of(0))
.build();
PlanChecker.from(connectContext, plan)
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java
index 4fd436c54a0..73b2fd2b5b1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushFilterInsideJoinTest.java
@@ -108,7 +108,7 @@ class PushFilterInsideJoinTest implements MemoPatternMatchSupported {
@Test
public void testShouldNotPushInsideJoin() {
for (JoinType joinType : JoinType.values()) {
- if (JoinType.INNER_JOIN == joinType || JoinType.CROSS_JOIN == joinType) {
+ if (joinType == JoinType.CROSS_JOIN || joinType == JoinType.INNER_JOIN) {
continue;
}
shouldNotPushInsideJoin(joinType);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java
index 71efbb42ff1..c8d8e8af7af 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ReorderJoinTest.java
@@ -43,15 +43,19 @@ class ReorderJoinTest implements MemoPatternMatchSupported {
@Test
public void testLeftOuterJoin() {
+ testLeftOuterJoinHelper(JoinType.LEFT_OUTER_JOIN);
+ }
+
+ private void testLeftOuterJoinHelper(JoinType joinType) {
ImmutableList<LogicalPlan> plans = ImmutableList.of(
new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.joinEmptyOn(scan3, JoinType.CROSS_JOIN)
.filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0)))
.build(),
new LogicalPlanBuilder(scan1)
.joinEmptyOn(scan3, JoinType.CROSS_JOIN)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0)))
.build()
);
@@ -61,15 +65,19 @@ class ReorderJoinTest implements MemoPatternMatchSupported {
@Test
public void testRightOuterJoin() {
+ testRightOuterJoinHelper(JoinType.RIGHT_OUTER_JOIN);
+ }
+
+ private void testRightOuterJoinHelper(JoinType joinType) {
ImmutableList<LogicalPlan> plans = ImmutableList.of(
new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.joinEmptyOn(scan3, JoinType.CROSS_JOIN)
.filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0)))
.build(),
new LogicalPlanBuilder(scan1)
.joinEmptyOn(scan3, JoinType.CROSS_JOIN)
- .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0))
+ .join(scan2, joinType, Pair.of(0, 0))
.filter(new EqualTo(scan3.getOutput().get(0), scan1.getOutput().get(0)))
.build()
);
@@ -248,11 +256,44 @@ class ReorderJoinTest implements MemoPatternMatchSupported {
logicalJoin(
logicalJoin(
logicalJoin().whenNot(join -> join.getJoinType().isCrossJoin()),
- leafPlan()
- ).whenNot(join -> join.getJoinType().isCrossJoin()),
- leafPlan()
- ).whenNot(join -> join.getJoinType().isCrossJoin())
- )
+ leafPlan()).whenNot(join -> join.getJoinType().isCrossJoin()),
+ leafPlan()).whenNot(join -> join.getJoinType().isCrossJoin()))
+ .printlnTree();
+ }
+
+ @Test
+ public void testAsofJoin() {
+ testAsofJoinHelper(JoinType.ASOF_LEFT_INNER_JOIN);
+ testAsofJoinHelper(JoinType.ASOF_RIGHT_INNER_JOIN);
+ testAsofJoinHelper(JoinType.ASOF_LEFT_OUTER_JOIN);
+ testAsofJoinHelper(JoinType.ASOF_RIGHT_OUTER_JOIN);
+ }
+
+ private void testAsofJoinHelper(JoinType joinType) {
+ LogicalPlan leftJoin = new LogicalPlanBuilder(scan1)
+ .join(scan2, joinType, Pair.of(0, 0))
+ .build();
+ LogicalPlan rightJoin = new LogicalPlanBuilder(scan3)
+ .join(scan4, joinType, Pair.of(0, 0))
+ .build();
+
+ LogicalPlan plan = new LogicalPlanBuilder(leftJoin)
+ .joinEmptyOn(rightJoin, JoinType.CROSS_JOIN)
+ .filter(new EqualTo(scan1.getOutput().get(0), scan3.getOutput().get(0)))
+ .build();
+ ConnectContext connectContext = MemoTestUtils.createConnectContext();
+ connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+ PlanChecker.from(connectContext, plan)
+ .applyBottomUp(new ReorderJoin())
+ .matchesFromRoot(
+ logicalJoin(
+ logicalJoin(
+ leafPlan(),
+ leafPlan()).whenNot(join -> join.getJoinType().isCrossJoin()),
+ logicalJoin(
+ leafPlan(),
+ leafPlan()).whenNot(join -> join.getJoinType().isCrossJoin()))
+ .whenNot(join -> join.getJoinType().isCrossJoin()))
.printlnTree();
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java
index 7fcb80862a3..499f0a34d88 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SaltJoinTest.java
@@ -31,8 +31,8 @@ public class SaltJoinTest extends TestWithFeService implements MemoPatternMatchS
@Override
protected void runBeforeAll() throws Exception {
createDatabase("test");
- createTable("create table test.test_skew9(a int,c varchar(100), b int) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");");
- createTable("create table test.test_skew10(a int,c varchar(100), b int) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");");
+ createTable("create table test.test_skew9(a int,c varchar(100), b int, d date) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");");
+ createTable("create table test.test_skew10(a int,c varchar(100), b int, d date) distributed by hash(a) buckets 32 properties(\"replication_num\"=\"1\");");
connectContext.setDatabase("test");
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
connectContext.getSessionVariable().setParallelResultSink(false);
@@ -59,6 +59,27 @@ public class SaltJoinTest extends TestWithFeService implements MemoPatternMatchS
);
}
+ @Test
+ public void testAsofInnerJoin() {
+ PlanChecker.from(connectContext)
+ .analyze("select * from test_skew9 tl asof inner join [shuffle[skew(tl.b(1,2))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b")
+ .rewrite()
+ .printlnTree()
+ .matches(
+ logicalJoin(
+ logicalProject(
+ logicalOlapScan()),
+ logicalProject(
+ logicalJoin(
+ logicalProject(
+ logicalGenerate(
+ logicalUnion())),
+ logicalOlapScan()
+ ).when(join -> join.getJoinType() == JoinType.RIGHT_OUTER_JOIN))
+ ).when(join -> join.getHashJoinConjuncts().size() == 2 && join.getJoinType() == JoinType.ASOF_LEFT_INNER_JOIN)
+ );
+ }
+
@Test
public void testLeftJoin() {
PlanChecker.from(connectContext)
@@ -80,6 +101,27 @@ public class SaltJoinTest extends TestWithFeService implements MemoPatternMatchS
);
}
+ @Test
+ public void testAsofLeftJoin() {
+ PlanChecker.from(connectContext)
+ .analyze("select * from test_skew9 tl asof left join [shuffle[skew(tl.b(1,2))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b;")
+ .rewrite()
+ .printlnTree()
+ .matches(
+ logicalJoin(
+ logicalProject(
+ logicalOlapScan()),
+ logicalProject(
+ logicalJoin(
+ logicalProject(
+ logicalGenerate(
+ logicalUnion())),
+ logicalOlapScan()
+ ).when(join -> join.getJoinType() == JoinType.RIGHT_OUTER_JOIN))
+ ).when(join -> join.getHashJoinConjuncts().size() == 2 && join.getJoinType() == JoinType.ASOF_LEFT_OUTER_JOIN)
+ );
+ }
+
@Test
public void testRightJoin() {
PlanChecker.from(connectContext)
@@ -146,6 +188,20 @@ public class SaltJoinTest extends TestWithFeService implements MemoPatternMatchS
);
}
+ @Test
+ void testAsofInnerSkewValueIsNull() {
+ PlanChecker.from(connectContext)
+ .analyze("select * from test_skew9 tl asof inner join [shuffle[skew(tl.b(null))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b")
+ .rewrite()
+ .printlnTree()
+ .matches(
+ logicalJoin(
+ logicalOlapScan(),
+ logicalOlapScan()
+ ).when(join -> join.getHashJoinConjuncts().size() == 1 && join.getJoinType() == JoinType.ASOF_LEFT_INNER_JOIN)
+ );
+ }
+
@Test void testLeftJoinSkewValueIsNull() {
PlanChecker.from(connectContext)
.analyze("select * from test_skew9 tl left join [shuffle[skew(tl.b(null))]] test_skew10 tr on tl.b = tr.b")
@@ -162,6 +218,23 @@ public class SaltJoinTest extends TestWithFeService implements MemoPatternMatchS
);
}
+ @Test
+ void testAsofLeftJoinSkewValueIsNull() {
+ PlanChecker.from(connectContext)
+ .analyze("select * from test_skew9 tl asof left join [shuffle[skew(tl.b(null))]] test_skew10 tr match_condition(tl.d > tr.d) on tl.b = tr.b")
+ .rewrite()
+ .printlnTree()
+ .matches(
+ logicalJoin(
+ logicalProject(
+ logicalOlapScan()),
+ logicalProject(
+ logicalOlapScan()
+ )
+ ).when(join -> join.getHashJoinConjuncts().size() == 2 && join.getJoinType() == JoinType.ASOF_LEFT_OUTER_JOIN)
+ );
+ }
+
@Test void testRightJoinSkewValueIsNull() {
PlanChecker.from(connectContext)
.analyze("select * from test_skew9 tl right join [shuffle[skew(tr.b(null))]] test_skew10 tr on tl.b = tr.b")
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java
index 8e37234a0c6..32ada369213 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/JoinEstimateTest.java
@@ -141,4 +141,46 @@ public class JoinEstimateTest {
Assertions.assertNotNull(outAStats);
Assertions.assertEquals(20.0, outCStats.ndv);
}
+
+ @Test
+ public void testAsofInnerJoinStats() {
+ SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+ SlotReference b = new SlotReference("b", IntegerType.INSTANCE);
+ EqualTo eq = new EqualTo(a, b);
+
+ // left probe
+ Statistics leftStats = new StatisticsBuilder().setRowCount(100).build();
+ leftStats.addColumnStats(a,
+ new ColumnStatisticBuilder(100)
+ .setNdv(50)
+ .build()
+ );
+
+ // right build with original ndv larger than current ndv
+ Statistics rightStats = new StatisticsBuilder().setRowCount(80).build();
+ ColumnStatistic original = new ColumnStatisticBuilder(80)
+ .setNdv(20)
+ .build();
+ rightStats.addColumnStats(b,
+ new ColumnStatisticBuilder(80)
+ .setNdv(10)
+ .setOriginal(original)
+ .build()
+ );
+
+ IdGenerator<GroupId> idGenerator = GroupId.createGenerator();
+ GroupPlan left = new GroupPlan(new Group(idGenerator.getNextId(), new LogicalProperties(
+ (Supplier<List<Slot>>) () -> Lists.newArrayList(a), () -> DataTrait.EMPTY_TRAIT)));
+ GroupPlan right = new GroupPlan(new Group(idGenerator.getNextId(), new LogicalProperties(
+ (Supplier<List<Slot>>) () -> Lists.newArrayList(b), () -> DataTrait.EMPTY_TRAIT)));
+
+ LogicalJoin join = new LogicalJoin(JoinType.ASOF_LEFT_INNER_JOIN, Lists.newArrayList(eq),
+ left, right, null);
+ Statistics outputStats = JoinEstimation.estimate(leftStats, rightStats, join);
+
+ // expected rowCount = left.rowCount * build.ndv / build.originalNdv = 100 * 10 / 20 = 50
+ Assertions.assertEquals(50.0, outputStats.getRowCount());
+ ColumnStatistic outAStats = outputStats.findColumnStatistics(a);
+ Assertions.assertNotNull(outAStats);
+ }
}
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index e87bf5e574b..0f9ccbf8b01 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -883,7 +883,11 @@ enum TJoinOp {
// be rejected (ANTI-join), based on the other join conjuncts. This is in contrast
// to LEFT_ANTI_JOIN where NULLs are not matches and therefore always returned.
NULL_AWARE_LEFT_ANTI_JOIN = 10,
- NULL_AWARE_LEFT_SEMI_JOIN = 11
+ NULL_AWARE_LEFT_SEMI_JOIN = 11,
+ ASOF_LEFT_INNER_JOIN = 12,
+ ASOF_RIGHT_INNER_JOIN = 13,
+ ASOF_LEFT_OUTER_JOIN = 14,
+ ASOF_RIGHT_OUTER_JOIN = 15
}
enum TJoinDistributionType {
@@ -928,6 +932,7 @@ struct THashJoinNode {
// use_specific_projections true, if output exprssions is denoted by srcExprList represents, o.w. PlanNode.projections
// deprecated
14: optional bool use_specific_projections
+ 15: optional Exprs.TExpr match_condition
}
struct TNestedLoopJoinNode {
diff --git a/regression-test/data/nereids_p0/join/asof/test_asof_join.out b/regression-test/data/nereids_p0/join/asof/test_asof_join.out
new file mode 100644
index 00000000000..94e8d260c36
--- /dev/null
+++ b/regression-test/data/nereids_p0/join/asof/test_asof_join.out
@@ -0,0 +1,327 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !asof_basic_ge --
+1 AAPL 2024-01-01T10:00:05 150.50 1 2024-01-01T10:00 150.00 150.10
+2 AAPL 2024-01-01T10:00:15 151.00 2 2024-01-01T10:00:10 150.40 150.60
+3 AAPL 2024-01-01T10:00:25 150.75 3 2024-01-01T10:00:20 150.90 151.10
+4 GOOG 2024-01-01T10:00:10 2800.00 4 2024-01-01T10:00:05 2795.00 2800.00
+5 GOOG 2024-01-01T10:00:20 2805.00 5 2024-01-01T10:00:15 2802.00 2808.00
+6 MSFT 2024-01-01T10:00:08 380.00 6 2024-01-01T10:00 378.00 380.00
+
+-- !asof_basic_le --
+1 AAPL 2024-01-01T10:00:05 150.50 2 2024-01-01T10:00:10 150.40
+2 AAPL 2024-01-01T10:00:15 151.00 3 2024-01-01T10:00:20 150.90
+3 AAPL 2024-01-01T10:00:25 150.75 \N \N \N
+4 GOOG 2024-01-01T10:00:10 2800.00 5 2024-01-01T10:00:15 2802.00
+5 GOOG 2024-01-01T10:00:20 2805.00 \N \N \N
+6 MSFT 2024-01-01T10:00:08 380.00 7 2024-01-01T10:00:10 379.50
+
+-- !asof_strict_gt --
+1 AAPL 2024-01-01T10:00:05 1 2024-01-01T10:00
+2 AAPL 2024-01-01T10:00:15 2 2024-01-01T10:00:10
+3 AAPL 2024-01-01T10:00:25 3 2024-01-01T10:00:20
+4 GOOG 2024-01-01T10:00:10 4 2024-01-01T10:00:05
+5 GOOG 2024-01-01T10:00:20 5 2024-01-01T10:00:15
+6 MSFT 2024-01-01T10:00:08 6 2024-01-01T10:00
+
+-- !asof_strict_lt --
+1 AAPL 2024-01-01T10:00:05 2 2024-01-01T10:00:10
+2 AAPL 2024-01-01T10:00:15 3 2024-01-01T10:00:20
+3 AAPL 2024-01-01T10:00:25 \N \N
+4 GOOG 2024-01-01T10:00:10 5 2024-01-01T10:00:15
+5 GOOG 2024-01-01T10:00:20 \N \N
+6 MSFT 2024-01-01T10:00:08 7 2024-01-01T10:00:10
+
+-- !asof_inner --
+1 AAPL 2024-01-01T10:00:05 150.50 1 2024-01-01T10:00 150.00
+2 AAPL 2024-01-01T10:00:15 151.00 2 2024-01-01T10:00:10 150.40
+3 AAPL 2024-01-01T10:00:25 150.75 3 2024-01-01T10:00:20 150.90
+4 GOOG 2024-01-01T10:00:10 2800.00 4 2024-01-01T10:00:05 2795.00
+5 GOOG 2024-01-01T10:00:20 2805.00 5 2024-01-01T10:00:15 2802.00
+6 MSFT 2024-01-01T10:00:08 380.00 6 2024-01-01T10:00 378.00
+
+-- !asof_events --
+1 1 2024-01-01T10:00:05 event_A 1 2024-01-01T10:00 snap_1
+2 1 2024-01-01T10:00:15 event_B 2 2024-01-01T10:00:10 snap_2
+3 1 2024-01-01T10:00:25 event_C 3 2024-01-01T10:00:20 snap_3
+4 2 2024-01-01T10:00:10 event_D 4 2024-01-01T10:00:05 snap_4
+5 2 2024-01-01T10:00:20 event_E 5 2024-01-01T10:00:15 snap_5
+6 3 2024-01-01T10:00:08 event_F \N \N \N
+
+-- !asof_no_match --
+1 1 2024-01-01T10:00:10 \N \N
+2 2 2024-01-01T10:00:20 \N \N
+
+-- !asof_empty_right --
+1 1 2024-01-01T10:00:10 \N
+2 2 2024-01-01T10:00:20 \N
+
+-- !asof_multi_eq --
+1 1 A 2024-01-01T10:00:10 1 2024-01-01T10:00:05 data_1
+2 1 A 2024-01-01T10:00:20 2 2024-01-01T10:00:15 data_2
+3 1 B 2024-01-01T10:00:15 3 2024-01-01T10:00:10 data_3
+4 2 A 2024-01-01T10:00:25 4 2024-01-01T10:00:20 data_4
+
+-- !asof_null_eq --
+1 1 2024-01-01T10:00:10 1 2024-01-01T10:00:05
+2 \N 2024-01-01T10:00:20 \N \N
+3 2 2024-01-01T10:00:30 3 2024-01-01T10:00:25
+
+-- !asof_null_match --
+1 1 2024-01-01T10:00:10 1 2024-01-01T10:00:05
+2 1 \N \N \N
+3 1 2024-01-01T10:00:30 3 2024-01-01T10:00:25
+
+-- !asof_single_match --
+1 2024-01-01T10:00:10 1 2024-01-01T10:00:05
+
+-- !asof_single_no_match --
+1 2024-01-01T10:00:10 \N \N
+
+-- !asof_many_ge --
+1 2024-01-01T10:00:50 8 2024-01-01T10:00:50
+
+-- !asof_many_gt --
+1 2024-01-01T10:00:50 7 2024-01-01T10:00:49
+
+-- !asof_many_le --
+1 2024-01-01T10:00:50 8 2024-01-01T10:00:50
+
+-- !asof_many_lt --
+1 2024-01-01T10:00:50 9 2024-01-01T10:00:51
+
+-- !asof_subquery --
+2 AAPL 2 150.40
+3 AAPL 3 150.90
+4 GOOG 4 2795.00
+5 GOOG 5 2802.00
+6 MSFT 6 378.00
+
+-- !asof_agg --
+AAPL 3 150.4333
+GOOG 2 2798.5000
+MSFT 1 378.0000
+
+-- !asof_filter --
+1 AAPL 2024-01-01T10:00:05 1 2024-01-01T10:00
+2 AAPL 2024-01-01T10:00:15 2 2024-01-01T10:00:10
+3 AAPL 2024-01-01T10:00:25 3 2024-01-01T10:00:20
+4 GOOG 2024-01-01T10:00:10 4 2024-01-01T10:00:05
+5 GOOG 2024-01-01T10:00:20 5 2024-01-01T10:00:15
+6 MSFT 2024-01-01T10:00:08 6 2024-01-01T10:00
+
+-- !asof_inner_filter --
+
+-- !asof_multi_level_3 --
+1 100 2024-01-01T10:00 5 12.00 2024-01-01T09:30 80 2024-01-01T09:00
+2 100 2024-01-01T11:00 3 11.50 2024-01-01T10:30 50 2024-01-01T10:30
+3 100 2024-01-01T10:30 2 14.00 2024-01-01T10:00 200 2024-01-01T08:00
+4 200 2024-01-01T10:15 10 25.00 2024-01-01T08:00 500 2024-01-01T08:00
+5 200 2024-01-01T12:00 7 27.00 2024-01-01T11:00 400 2024-01-01T11:30
+6 100 2024-01-01T09:00 1 10.00 2024-01-01T08:00 80 2024-01-01T09:00
+
+-- !asof_multi_level_4 --
+1 100 2024-01-01T10:00 12.00 80 10
+2 100 2024-01-01T11:00 11.50 50 15
+3 100 2024-01-01T10:30 14.00 200 15
+4 200 2024-01-01T10:15 25.00 500 5
+5 200 2024-01-01T12:00 27.00 400 5
+6 100 2024-01-01T09:00 10.00 80 10
+
+-- !asof_with_regular_join --
+1 Widget A Electronics 2024-01-01T10:00 12.00 2024-01-01T09:30
+2 Widget A Electronics 2024-01-01T11:00 11.50 2024-01-01T10:30
+3 Widget A Electronics 2024-01-01T10:30 14.00 2024-01-01T10:00
+4 Widget B Electronics 2024-01-01T10:15 25.00 2024-01-01T08:00
+5 Widget B Electronics 2024-01-01T12:00 27.00 2024-01-01T11:00
+6 Widget A Electronics 2024-01-01T09:00 10.00 2024-01-01T08:00
+
+-- !asof_expr_key --
+1 10 5 2024-01-01T10:00 2 2024-01-01T09:30 data_b
+2 20 10 2024-01-01T11:00 5 2024-01-01T10:30 data_e
+3 15 5 2024-01-01T12:00 6 2024-01-01T11:00 data_f
+
+-- !asof_str_expr --
+1 ABC-001 2024-01-01T10:00 2 2024-01-01T09:30 110
+2 ABC-002 2024-01-01T11:00 3 2024-01-01T10:30 120
+3 XYZ-001 2024-01-01T12:00 5 2024-01-01T11:30 210
+
+-- !asof_self_join --
+1 2024-01-01T10:00 6 2024-01-01T09:00
+2 2024-01-01T11:00 1 2024-01-01T10:00
+3 2024-01-01T10:30 \N \N
+4 2024-01-01T10:15 \N \N
+5 2024-01-01T12:00 4 2024-01-01T10:15
+6 2024-01-01T09:00 \N \N
+
+-- !asof_date --
+1 2024-01-10 2 2024-01-08 mid
+2 2024-01-15 3 2024-01-12 late
+3 2024-01-20 4 2024-01-18 data_x
+
+-- !asof_complex_agg --
+100 EU 1 2 14.0000 200
+100 US 3 9 11.1666 50
+200 US 2 17 26.0000 400
+
+-- !asof_case_expr --
+1 LOW 60.00
+2 LOW 34.50
+3 LOW 28.00
+4 HIGH 250.00
+5 HIGH 189.00
+6 LOW 10.00
+
+-- !asof_mixed_directions --
+1 2024-01-01T10:00 12.00 2024-01-01T09:30 11.50 2024-01-01T10:30
+2 2024-01-01T11:00 11.50 2024-01-01T10:30 \N \N
+6 2024-01-01T09:00 10.00 2024-01-01T08:00 12.00 2024-01-01T09:30
+
+-- !asof_boundary_gt --
+1 2024-01-01T10:00 2 2024-01-01T09:00 before
+2 2024-01-01T10:00:01 1 2024-01-01T10:00 exact
+3 2024-01-01T09:59:59 2 2024-01-01T09:00 before
+
+-- !asof_boundary_lt --
+1 2024-01-01T10:00 3 2024-01-01T11:00 after
+2 2024-01-01T10:00:01 3 2024-01-01T11:00 after
+3 2024-01-01T09:59:59 1 2024-01-01T10:00 exact
+
+-- !asof_boundary_ge --
+1 2024-01-01T10:00 1 2024-01-01T10:00 exact
+2 2024-01-01T10:00:01 1 2024-01-01T10:00 exact
+3 2024-01-01T09:59:59 2 2024-01-01T09:00 before
+
+-- !asof_boundary_le --
+1 2024-01-01T10:00 1 2024-01-01T10:00 exact
+2 2024-01-01T10:00:01 3 2024-01-01T11:00 after
+3 2024-01-01T09:59:59 1 2024-01-01T10:00 exact
+
+-- !asof_all_filtered --
+1 2024-01-01T08:00 \N \N
+
+-- !asof_multi_grp --
+1 1 2024-01-01T10:00 3 2024-01-01T09:45 120
+2 2 2024-01-01T10:00 \N \N \N
+3 3 2024-01-01T10:00 5 2024-01-01T10:00 300
+4 4 2024-01-01T10:00 \N \N \N
+
+-- !asof_precision --
+1 2024-01-01T10:00:00.000500 2 2024-01-01T10:00:00.000400 200
+2 2024-01-01T10:00:00.001 4 2024-01-01T10:00:00.001 400
+
+-- !asof_left_inner_ge --
+3 1 2024-01-01T10:00 2024-01-01T10:00 1
+4 1 2024-01-01T11:00 2024-01-01T10:00 1
+5 1 2024-01-01T12:00 2024-01-01T12:00 2
+6 1 2024-01-01T13:00 2024-01-01T12:00 2
+9 2 2024-01-01T11:00 2024-01-01T11:00 3
+10 2 2024-01-01T12:00 2024-01-01T11:00 3
+12 1 2024-01-01T10:30 2024-01-01T10:00 1
+13 1 2024-01-01T11:30 2024-01-01T10:00 1
+16 2 2024-01-01T11:30 2024-01-01T11:00 3
+17 1 2024-01-01T14:00 2024-01-01T12:00 2
+18 2 2024-01-01T13:00 2024-01-01T11:00 3
+
+-- !asof_left_outer_ge --
+1 1 2024-01-01T08:00 \N \N
+2 1 2024-01-01T09:00 \N \N
+3 1 2024-01-01T10:00 2024-01-01T10:00 1
+4 1 2024-01-01T11:00 2024-01-01T10:00 1
+5 1 2024-01-01T12:00 2024-01-01T12:00 2
+6 1 2024-01-01T13:00 2024-01-01T12:00 2
+7 2 2024-01-01T09:00 \N \N
+8 2 2024-01-01T10:00 \N \N
+9 2 2024-01-01T11:00 2024-01-01T11:00 3
+10 2 2024-01-01T12:00 2024-01-01T11:00 3
+11 1 2024-01-01T09:30 \N \N
+12 1 2024-01-01T10:30 2024-01-01T10:00 1
+13 1 2024-01-01T11:30 2024-01-01T10:00 1
+14 2 2024-01-01T09:30 \N \N
+15 2 2024-01-01T10:30 \N \N
+16 2 2024-01-01T11:30 2024-01-01T11:00 3
+17 1 2024-01-01T14:00 2024-01-01T12:00 2
+18 2 2024-01-01T13:00 2024-01-01T11:00 3
+19 3 2024-01-01T10:00 \N \N
+20 3 2024-01-01T11:00 \N \N
+
+-- !asof_expr_condition --
+3 1 2024-01-01T10:00 2024-01-01T10:00 1
+4 1 2024-01-01T11:00 2024-01-01T10:00 1
+5 1 2024-01-01T12:00 2024-01-01T12:00 2
+6 1 2024-01-01T13:00 2024-01-01T12:00 2
+9 2 2024-01-01T11:00 2024-01-01T11:00 3
+10 2 2024-01-01T12:00 2024-01-01T11:00 3
+12 1 2024-01-01T10:30 2024-01-01T10:00 1
+13 1 2024-01-01T11:30 2024-01-01T10:00 1
+16 2 2024-01-01T11:30 2024-01-01T11:00 3
+17 1 2024-01-01T14:00 2024-01-01T12:00 2
+18 2 2024-01-01T13:00 2024-01-01T11:00 3
+
+-- !asof_bsearch_ge --
+3 1 2024-01-01T10:00 2024-01-01T10:00 1
+4 1 2024-01-01T11:00 2024-01-01T10:00 1
+5 1 2024-01-01T12:00 2024-01-01T12:00 2
+6 1 2024-01-01T13:00 2024-01-01T12:00 2
+9 2 2024-01-01T11:00 2024-01-01T11:00 3
+10 2 2024-01-01T12:00 2024-01-01T11:00 3
+12 1 2024-01-01T10:30 2024-01-01T10:00 1
+13 1 2024-01-01T11:30 2024-01-01T10:00 1
+16 2 2024-01-01T11:30 2024-01-01T11:00 3
+17 1 2024-01-01T14:00 2024-01-01T12:00 2
+18 2 2024-01-01T13:00 2024-01-01T11:00 3
+
+-- !asof_bsearch_le --
+1 1 2024-01-01T08:00 2024-01-01T10:00 1
+2 1 2024-01-01T09:00 2024-01-01T10:00 1
+3 1 2024-01-01T10:00 2024-01-01T10:00 1
+4 1 2024-01-01T11:00 2024-01-01T12:00 2
+5 1 2024-01-01T12:00 2024-01-01T12:00 2
+7 2 2024-01-01T09:00 2024-01-01T11:00 3
+8 2 2024-01-01T10:00 2024-01-01T11:00 3
+9 2 2024-01-01T11:00 2024-01-01T11:00 3
+11 1 2024-01-01T09:30 2024-01-01T10:00 1
+12 1 2024-01-01T10:30 2024-01-01T12:00 2
+13 1 2024-01-01T11:30 2024-01-01T12:00 2
+14 2 2024-01-01T09:30 2024-01-01T11:00 3
+15 2 2024-01-01T10:30 2024-01-01T11:00 3
+
+-- !asof_bsearch_gt --
+4 1 2024-01-01T11:00 2024-01-01T10:00 1
+5 1 2024-01-01T12:00 2024-01-01T10:00 1
+6 1 2024-01-01T13:00 2024-01-01T12:00 2
+10 2 2024-01-01T12:00 2024-01-01T11:00 3
+12 1 2024-01-01T10:30 2024-01-01T10:00 1
+13 1 2024-01-01T11:30 2024-01-01T10:00 1
+16 2 2024-01-01T11:30 2024-01-01T11:00 3
+17 1 2024-01-01T14:00 2024-01-01T12:00 2
+18 2 2024-01-01T13:00 2024-01-01T11:00 3
+
+-- !asof_bsearch_lt --
+1 1 2024-01-01T08:00 2024-01-01T10:00 1
+2 1 2024-01-01T09:00 2024-01-01T10:00 1
+3 1 2024-01-01T10:00 2024-01-01T12:00 2
+4 1 2024-01-01T11:00 2024-01-01T12:00 2
+7 2 2024-01-01T09:00 2024-01-01T11:00 3
+8 2 2024-01-01T10:00 2024-01-01T11:00 3
+11 1 2024-01-01T09:30 2024-01-01T10:00 1
+12 1 2024-01-01T10:30 2024-01-01T12:00 2
+13 1 2024-01-01T11:30 2024-01-01T12:00 2
+14 2 2024-01-01T09:30 2024-01-01T11:00 3
+15 2 2024-01-01T10:30 2024-01-01T11:00 3
+
+-- !asof_multi_grp_outer --
+1 1 2024-01-01T10:00 2024-01-01T10:00 110
+2 2 2024-01-01T10:00 \N \N
+3 3 2024-01-01T10:00 2024-01-01T10:00 310
+4 4 2024-01-01T10:00 \N \N
+5 1 2024-01-01T11:00 2024-01-01T11:00 120
+6 2 2024-01-01T09:00 \N \N
+7 3 2024-01-01T11:00 2024-01-01T10:00 310
+8 1 2024-01-01T09:00 2024-01-01T09:00 100
+9 2 2024-01-01T11:00 2024-01-01T11:00 200
+10 3 2024-01-01T08:00 \N \N
+
+-- !asof_timestamptz --
+1 2024-01-01 10:00:00+08:00 2 2024-01-01 10:00:00+08:00 200
+2 2024-01-01 12:00:00+08:00 3 2024-01-01 11:00:00+08:00 300
+
diff --git a/regression-test/data/nereids_p0/join/asof/test_asof_join_build_expr_edge.out b/regression-test/data/nereids_p0/join/asof/test_asof_join_build_expr_edge.out
new file mode 100644
index 00000000000..38ba85a8c97
--- /dev/null
+++ b/regression-test/data/nereids_p0/join/asof/test_asof_join_build_expr_edge.out
@@ -0,0 +1,79 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !abee_cte_expr --
+1 2 110
+2 4 130
+3 6 210
+4 8 230
+5 9 300
+
+-- !abee_inline_view --
+1 2 110
+2 4 130
+3 6 210
+4 8 230
+5 9 300
+
+-- !abee_join_build --
+1 2 110 alpha
+2 4 130 alpha
+3 6 210 beta
+4 8 230 beta
+5 9 300 gamma
+
+-- !abee_chained --
+1 3 120 2 b2_2
+2 4 130 2 b2_2
+3 6 210 3 b2_3
+4 8 230 4 b2_4
+5 10 310 5 b2_5
+
+-- !abee_compound_keys --
+1 2 110
+2 4 130
+3 6 210
+4 8 230
+5 9 300
+
+-- !abee_self_join --
+1 \N \N
+2 1 100
+3 2 110
+4 3 120
+5 \N \N
+6 5 200
+7 6 210
+8 7 220
+9 \N \N
+10 \N \N
+
+-- !abee_cross_precision --
+1 2 110
+2 3 120
+
+-- !abee_agg_build --
+1 A 30
+2 B 70
+3 \N \N
+4 B 150
+5 \N \N
+
+-- !abee_where_filter --
+3 6 210
+4 8 230
+5 9 300
+
+-- !abee_nested_subquery --
+1 2 110
+2 4 130
+3 6 210
+4 8 230
+5 9 300
+
+-- !abee_inner_cte --
+2 1 100
+3 2 110
+4 3 120
+6 5 200
+7 6 210
+8 7 220
+
diff --git a/regression-test/data/nereids_p0/join/asof/test_asof_join_explicit.out b/regression-test/data/nereids_p0/join/asof/test_asof_join_explicit.out
new file mode 100644
index 00000000000..0f3b9f48524
--- /dev/null
+++ b/regression-test/data/nereids_p0/join/asof/test_asof_join_explicit.out
@@ -0,0 +1,97 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !left_outer_ge --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !right_outer_ge --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !left_inner_ge --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !right_inner_ge --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !left_outer_le --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
+-- !right_outer_le --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
+-- !left_outer_gt --
+1 1 2024-01-01T10:00 2024-01-01T09:30 115
+2 1 2024-01-01T12:00 2024-01-01T11:30 135
+3 2 2024-01-01T11:00 2024-01-01T10:30 215
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !right_outer_gt --
+1 1 2024-01-01T10:00 2024-01-01T09:30 115
+2 1 2024-01-01T12:00 2024-01-01T11:30 135
+3 2 2024-01-01T11:00 2024-01-01T10:30 215
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !left_outer_lt --
+1 1 2024-01-01T10:00 2024-01-01T10:30 125
+2 1 2024-01-01T12:00 2024-01-01T13:00 150
+3 2 2024-01-01T11:00 2024-01-01T11:30 225
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
+-- !right_outer_lt --
+1 1 2024-01-01T10:00 2024-01-01T10:30 125
+2 1 2024-01-01T12:00 2024-01-01T13:00 150
+3 2 2024-01-01T11:00 2024-01-01T11:30 225
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
+-- !left_inner_le --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
+-- !right_inner_le --
+1 1 2024-01-01T10:00 2024-01-01T10:00 120
+2 1 2024-01-01T12:00 2024-01-01T12:00 140
+3 2 2024-01-01T11:00 2024-01-01T11:00 220
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
+-- !left_inner_gt --
+1 1 2024-01-01T10:00 2024-01-01T09:30 115
+2 1 2024-01-01T12:00 2024-01-01T11:30 135
+3 2 2024-01-01T11:00 2024-01-01T10:30 215
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !right_inner_gt --
+1 1 2024-01-01T10:00 2024-01-01T09:30 115
+2 1 2024-01-01T12:00 2024-01-01T11:30 135
+3 2 2024-01-01T11:00 2024-01-01T10:30 215
+4 3 2024-01-01T10:00 2024-01-01T09:00 300
+
+-- !left_inner_lt --
+1 1 2024-01-01T10:00 2024-01-01T10:30 125
+2 1 2024-01-01T12:00 2024-01-01T13:00 150
+3 2 2024-01-01T11:00 2024-01-01T11:30 225
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
+-- !right_inner_lt --
+1 1 2024-01-01T10:00 2024-01-01T10:30 125
+2 1 2024-01-01T12:00 2024-01-01T13:00 150
+3 2 2024-01-01T11:00 2024-01-01T11:30 225
+4 3 2024-01-01T10:00 2024-01-01T11:00 310
+
diff --git a/regression-test/data/nereids_p0/join/asof/test_asof_join_expr.out b/regression-test/data/nereids_p0/join/asof/test_asof_join_expr.out
new file mode 100644
index 00000000000..d4a48604630
--- /dev/null
+++ b/regression-test/data/nereids_p0/join/asof/test_asof_join_expr.out
@@ -0,0 +1,107 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !expr_build_interval --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_probe_interval --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_simple --
+1 2024-01-01T10:00 3 2024-01-01T10:00 r3
+2 2024-01-01T11:00 4 2024-01-01T11:00 r4
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_date_add --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_date_sub --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_days_add --
+1 2024-01-01T10:00 3 2024-01-01T10:00 r3
+2 2024-01-01T11:00 4 2024-01-01T11:00 r4
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_nested_func --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_hours_add --
+1 2024-01-01T10:00 1 2024-01-01T08:00 r1
+2 2024-01-01T11:00 2 2024-01-01T09:00 r2
+3 2024-01-01T12:00 3 2024-01-01T10:00 r3
+
+-- !expr_multi_interval --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_large_dataset --
+1 2024-01-01T12:00 4 2024-01-01T09:00 r04
+2 2024-01-01T15:00 7 2024-01-01T12:00 r07
+3 2024-01-01T18:00 10 2024-01-01T15:00 r10
+
+-- !expr_large_strict --
+1 2024-01-01T12:00 3 2024-01-01T08:00 r03
+2 2024-01-01T15:00 6 2024-01-01T11:00 r06
+3 2024-01-01T18:00 9 2024-01-01T14:00 r09
+
+-- !expr_large_le --
+1 2024-01-01T12:00 10 2024-01-01T15:00 r10
+2 2024-01-01T15:00 13 2024-01-01T18:00 r13
+3 2024-01-01T18:00 \N \N \N
+
+-- !expr_large_lt --
+1 2024-01-01T12:00 11 2024-01-01T16:00 r11
+2 2024-01-01T15:00 14 2024-01-01T19:00 r14
+3 2024-01-01T18:00 \N \N \N
+
+-- !expr_no_match --
+1 2024-01-01T12:00 \N \N
+2 2024-01-01T15:00 \N \N
+3 2024-01-01T18:00 \N \N
+
+-- !expr_inner --
+1 2024-01-01T12:00 4 2024-01-01T09:00 r04
+2 2024-01-01T15:00 7 2024-01-01T12:00 r07
+3 2024-01-01T18:00 10 2024-01-01T15:00 r10
+
+-- !expr_multigrp --
+1 1 2024-01-01T10:00 2 2024-01-01T09:00 g1_r2
+2 2 2024-01-01T10:00 4 2024-01-01T08:30 g2_r2
+3 3 2024-01-01T10:00 \N \N \N
+
+-- !expr_both_sides --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_deep_nested --
+1 2024-01-01T10:00 2 2024-01-01T09:00 r2
+2 2024-01-01T11:00 3 2024-01-01T10:00 r3
+3 2024-01-01T12:00 4 2024-01-01T11:00 r4
+
+-- !expr_minutes_add --
+1 2024-01-01T10:00 1 2024-01-01T08:00 r1
+2 2024-01-01T11:00 2 2024-01-01T09:00 r2
+3 2024-01-01T12:00 3 2024-01-01T10:00 r3
+
+-- !expr_edge_exact --
+1 2024-01-01T10:00 1 2024-01-01T09:00 exact_match
+
+-- !expr_edge_strict --
+1 2024-01-01T10:00 2 2024-01-01T08:00 before
+
+-- !expr_null_probe --
+1 2024-01-01T10:00 3 2024-01-01T09:00 r3
+2 \N \N \N \N
+3 2024-01-01T12:00 3 2024-01-01T09:00 r3
+
diff --git a/regression-test/data/nereids_p0/join/asof/test_asof_join_large_bucket.out b/regression-test/data/nereids_p0/join/asof/test_asof_join_large_bucket.out
new file mode 100644
index 00000000000..6599dc8e2a9
--- /dev/null
+++ b/regression-test/data/nereids_p0/join/asof/test_asof_join_large_bucket.out
@@ -0,0 +1,35 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !lb_left_count --
+1 51
+2 51
+3 51
+
+-- !lb_inner_count --
+1 50
+2 50
+3 50
+
+-- !lb_spot_check --
+1 1 2024-01-01T10:00 1 2024-01-01T10:00 1001
+25 1 2024-01-01T10:24 13 2024-01-01T10:24 1013
+50 1 2024-01-01T10:49 25 2024-01-01T10:48 1025
+51 2 2024-01-01T10:00 31 2024-01-01T10:00 2031
+100 2 2024-01-01T10:49 55 2024-01-01T10:48 2055
+151 1 \N \N \N \N
+152 2 \N \N \N \N
+
+-- !lb_le_count --
+1 51
+2 51
+3 51
+
+-- !lb_gt_count --
+1 51
+2 51
+3 51
+
+-- !lb_lt_count --
+1 51
+2 51
+3 51
+
diff --git a/regression-test/suites/nereids_p0/join/asof/test_asof_join.groovy b/regression-test/suites/nereids_p0/join/asof/test_asof_join.groovy
new file mode 100644
index 00000000000..6efc63dbbb6
--- /dev/null
+++ b/regression-test/suites/nereids_p0/join/asof/test_asof_join.groovy
@@ -0,0 +1,1621 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// ASOF JOIN Syntax:
+// FROM <left_table> ASOF [ INNER | LEFT ] JOIN <right_table>
+// MATCH_CONDITION ( <left_table.timecol> <comparison_operator> <right_table.timecol> )
+// ON <left_table.col> = <right_table.col> [ AND ... ]
+// comparison_operator: >=, <=, >, <
+
+suite("test_asof_join", "nereids_p0") {
+ // Create test tables for ASOF JOIN
+ sql "DROP TABLE IF EXISTS asof_trades"
+ sql "DROP TABLE IF EXISTS asof_quotes"
+ sql "DROP TABLE IF EXISTS asof_events"
+ sql "DROP TABLE IF EXISTS asof_snapshots"
+
+ // Table 1: trades - represents trade events with timestamps
+ sql """
+ CREATE TABLE asof_trades (
+ trade_id INT,
+ symbol VARCHAR(10),
+ trade_time DATETIME,
+ price DECIMAL(10, 2),
+ quantity INT
+ ) DISTRIBUTED BY HASH(trade_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Table 2: quotes - represents quote snapshots with timestamps
+ sql """
+ CREATE TABLE asof_quotes (
+ quote_id INT,
+ symbol VARCHAR(10),
+ quote_time DATETIME,
+ bid_price DECIMAL(10, 2),
+ ask_price DECIMAL(10, 2)
+ ) DISTRIBUTED BY HASH(quote_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Table 3: events - for testing with DATETIME match column
+ sql """
+ CREATE TABLE asof_events (
+ event_id INT,
+ category INT,
+ event_time DATETIME,
+ event_value VARCHAR(50)
+ ) DISTRIBUTED BY HASH(event_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Table 4: snapshots - for testing with DATETIME match column
+ sql """
+ CREATE TABLE asof_snapshots (
+ snapshot_id INT,
+ category INT,
+ snapshot_time DATETIME,
+ snapshot_data VARCHAR(50)
+ ) DISTRIBUTED BY HASH(snapshot_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Insert test data for trades
+ sql """
+ INSERT INTO asof_trades VALUES
+ (1, 'AAPL', '2024-01-01 10:00:05', 150.50, 100),
+ (2, 'AAPL', '2024-01-01 10:00:15', 151.00, 200),
+ (3, 'AAPL', '2024-01-01 10:00:25', 150.75, 150),
+ (4, 'GOOG', '2024-01-01 10:00:10', 2800.00, 50),
+ (5, 'GOOG', '2024-01-01 10:00:20', 2805.00, 75),
+ (6, 'MSFT', '2024-01-01 10:00:08', 380.00, 120)
+ """
+
+ // Insert test data for quotes
+ sql """
+ INSERT INTO asof_quotes VALUES
+ (1, 'AAPL', '2024-01-01 10:00:00', 150.00, 150.10),
+ (2, 'AAPL', '2024-01-01 10:00:10', 150.40, 150.60),
+ (3, 'AAPL', '2024-01-01 10:00:20', 150.90, 151.10),
+ (4, 'GOOG', '2024-01-01 10:00:05', 2795.00, 2800.00),
+ (5, 'GOOG', '2024-01-01 10:00:15', 2802.00, 2808.00),
+ (6, 'MSFT', '2024-01-01 10:00:00', 378.00, 380.00),
+ (7, 'MSFT', '2024-01-01 10:00:10', 379.50, 381.00)
+ """
+
+ // Insert test data for events
+ sql """
+ INSERT INTO asof_events VALUES
+ (1, 1, '2024-01-01 10:00:05', 'event_A'),
+ (2, 1, '2024-01-01 10:00:15', 'event_B'),
+ (3, 1, '2024-01-01 10:00:25', 'event_C'),
+ (4, 2, '2024-01-01 10:00:10', 'event_D'),
+ (5, 2, '2024-01-01 10:00:20', 'event_E'),
+ (6, 3, '2024-01-01 10:00:08', 'event_F')
+ """
+
+ // Insert test data for snapshots
+ sql """
+ INSERT INTO asof_snapshots VALUES
+ (1, 1, '2024-01-01 10:00:00', 'snap_1'),
+ (2, 1, '2024-01-01 10:00:10', 'snap_2'),
+ (3, 1, '2024-01-01 10:00:20', 'snap_3'),
+ (4, 2, '2024-01-01 10:00:05', 'snap_4'),
+ (5, 2, '2024-01-01 10:00:15', 'snap_5'),
+ (6, 3, '2024-01-01 10:00:10', 'snap_6')
+ """
+
+ // ==================== Test 1: Basic ASOF LEFT JOIN with >= ====================
+ // For each trade, find the most recent quote (quote_time <= trade_time)
+ qt_asof_basic_ge """
+ SELECT t.trade_id, t.symbol, t.trade_time, t.price,
+ q.quote_id, q.quote_time, q.bid_price, q.ask_price
+ FROM asof_trades t
+ ASOF LEFT JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // ==================== Test 2: Basic ASOF LEFT JOIN with <= ====================
+ // For each trade, find the next quote (quote_time >= trade_time)
+ qt_asof_basic_le """
+ SELECT t.trade_id, t.symbol, t.trade_time, t.price,
+ q.quote_id, q.quote_time, q.bid_price
+ FROM asof_trades t
+ ASOF LEFT JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time <= q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // ==================== Test 3: ASOF LEFT JOIN with > (strict greater) ====================
+ qt_asof_strict_gt """
+ SELECT t.trade_id, t.symbol, t.trade_time,
+ q.quote_id, q.quote_time
+ FROM asof_trades t
+ ASOF LEFT JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time > q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // ==================== Test 4: ASOF LEFT JOIN with < (strict less) ====================
+ qt_asof_strict_lt """
+ SELECT t.trade_id, t.symbol, t.trade_time,
+ q.quote_id, q.quote_time
+ FROM asof_trades t
+ ASOF LEFT JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time < q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // ==================== Test 5: ASOF INNER JOIN (no NULL for unmatched) ====================
+ qt_asof_inner """
+ SELECT t.trade_id, t.symbol, t.trade_time, t.price,
+ q.quote_id, q.quote_time, q.bid_price
+ FROM asof_trades t
+ ASOF INNER JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ ORDER BY t.trade_id
+ """
+
+ // ==================== Test 6: ASOF JOIN with another DATETIME example ====================
+ qt_asof_events """
+ SELECT e.event_id, e.category, e.event_time, e.event_value,
+ s.snapshot_id, s.snapshot_time, s.snapshot_data
+ FROM asof_events e
+ ASOF LEFT JOIN asof_snapshots s
+ MATCH_CONDITION(e.event_time >= s.snapshot_time)
+ ON e.category = s.category
+ ORDER BY e.event_id
+ """
+
+ // ==================== Test 7: No match case (LEFT JOIN should return NULL) ====================
+ sql "DROP TABLE IF EXISTS asof_no_match_left"
+ sql "DROP TABLE IF EXISTS asof_no_match_right"
+
+ sql """
+ CREATE TABLE asof_no_match_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_no_match_right (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_no_match_left VALUES
+ (1, 1, '2024-01-01 10:00:10'),
+ (2, 2, '2024-01-01 10:00:20')
+ """
+
+ sql """
+ INSERT INTO asof_no_match_right VALUES
+ (1, 1, '2024-01-01 10:01:00'),
+ (2, 3, '2024-01-01 10:00:30')
+ """
+
+ // Group 1: left.ts < right.ts, so no match for >=
+ // Group 2: no matching group in right table
+ qt_asof_no_match """
+ SELECT l.id, l.grp, l.ts, r.id as rid, r.ts as rts
+ FROM asof_no_match_left l
+ ASOF LEFT JOIN asof_no_match_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 8: Empty right table ====================
+ sql "DROP TABLE IF EXISTS asof_empty_right"
+ sql """
+ CREATE TABLE asof_empty_right (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ qt_asof_empty_right """
+ SELECT l.id, l.grp, l.ts, r.id as rid
+ FROM asof_no_match_left l
+ ASOF LEFT JOIN asof_empty_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 9: Multiple equality conditions ====================
+ sql "DROP TABLE IF EXISTS asof_multi_eq_left"
+ sql "DROP TABLE IF EXISTS asof_multi_eq_right"
+
+ sql """
+ CREATE TABLE asof_multi_eq_left (
+ id INT,
+ grp1 INT,
+ grp2 VARCHAR(10),
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_multi_eq_right (
+ id INT,
+ grp1 INT,
+ grp2 VARCHAR(10),
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_multi_eq_left VALUES
+ (1, 1, 'A', '2024-01-01 10:00:10'),
+ (2, 1, 'A', '2024-01-01 10:00:20'),
+ (3, 1, 'B', '2024-01-01 10:00:15'),
+ (4, 2, 'A', '2024-01-01 10:00:25')
+ """
+
+ sql """
+ INSERT INTO asof_multi_eq_right VALUES
+ (1, 1, 'A', '2024-01-01 10:00:05', 'data_1'),
+ (2, 1, 'A', '2024-01-01 10:00:15', 'data_2'),
+ (3, 1, 'B', '2024-01-01 10:00:10', 'data_3'),
+ (4, 2, 'A', '2024-01-01 10:00:20', 'data_4'),
+ (5, 2, 'A', '2024-01-01 10:00:30', 'data_5')
+ """
+
+ qt_asof_multi_eq """
+ SELECT l.id, l.grp1, l.grp2, l.ts,
+ r.id as rid, r.ts as rts, r.data
+ FROM asof_multi_eq_left l
+ ASOF LEFT JOIN asof_multi_eq_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp1 = r.grp1 AND l.grp2 = r.grp2
+ ORDER BY l.id
+ """
+
+ // ==================== Test 10: NULL values in equality column ====================
+ sql "DROP TABLE IF EXISTS asof_null_eq_left"
+ sql "DROP TABLE IF EXISTS asof_null_eq_right"
+
+ sql """
+ CREATE TABLE asof_null_eq_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_null_eq_right (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_null_eq_left VALUES
+ (1, 1, '2024-01-01 10:00:10'),
+ (2, NULL, '2024-01-01 10:00:20'),
+ (3, 2, '2024-01-01 10:00:30')
+ """
+
+ sql """
+ INSERT INTO asof_null_eq_right VALUES
+ (1, 1, '2024-01-01 10:00:05'),
+ (2, NULL, '2024-01-01 10:00:15'),
+ (3, 2, '2024-01-01 10:00:25')
+ """
+
+ qt_asof_null_eq """
+ SELECT l.id, l.grp, l.ts, r.id as rid, r.ts as rts
+ FROM asof_null_eq_left l
+ ASOF LEFT JOIN asof_null_eq_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 11: NULL values in match column ====================
+ sql "DROP TABLE IF EXISTS asof_null_match_left"
+ sql "DROP TABLE IF EXISTS asof_null_match_right"
+
+ sql """
+ CREATE TABLE asof_null_match_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_null_match_right (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_null_match_left VALUES
+ (1, 1, '2024-01-01 10:00:10'),
+ (2, 1, NULL),
+ (3, 1, '2024-01-01 10:00:30')
+ """
+
+ sql """
+ INSERT INTO asof_null_match_right VALUES
+ (1, 1, '2024-01-01 10:00:05'),
+ (2, 1, NULL),
+ (3, 1, '2024-01-01 10:00:25')
+ """
+
+ qt_asof_null_match """
+ SELECT l.id, l.grp, l.ts, r.id as rid, r.ts as rts
+ FROM asof_null_match_left l
+ ASOF LEFT JOIN asof_null_match_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 12: Single row tables ====================
+ sql "DROP TABLE IF EXISTS asof_single_left"
+ sql "DROP TABLE IF EXISTS asof_single_right"
+
+ sql """
+ CREATE TABLE asof_single_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_single_right (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql "INSERT INTO asof_single_left VALUES (1, 1, '2024-01-01 10:00:10')"
+ sql "INSERT INTO asof_single_right VALUES (1, 1, '2024-01-01 10:00:05')"
+
+ qt_asof_single_match """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_single_left l
+ ASOF LEFT JOIN asof_single_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ sql "TRUNCATE TABLE asof_single_right"
+ sql "INSERT INTO asof_single_right VALUES (1, 1, '2024-01-01 10:00:15')"
+
+ qt_asof_single_no_match """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_single_left l
+ ASOF LEFT JOIN asof_single_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 13: Large number of matches in same bucket ====================
+ sql "DROP TABLE IF EXISTS asof_many_matches_left"
+ sql "DROP TABLE IF EXISTS asof_many_matches_right"
+
+ sql """
+ CREATE TABLE asof_many_matches_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_many_matches_right (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql "INSERT INTO asof_many_matches_left VALUES (1, 1, '2024-01-01 10:00:50')"
+
+ sql """
+ INSERT INTO asof_many_matches_right VALUES
+ (1, 1, '2024-01-01 10:00:10'), (2, 1, '2024-01-01 10:00:20'),
+ (3, 1, '2024-01-01 10:00:30'), (4, 1, '2024-01-01 10:00:40'),
+ (5, 1, '2024-01-01 10:00:45'), (6, 1, '2024-01-01 10:00:48'),
+ (7, 1, '2024-01-01 10:00:49'), (8, 1, '2024-01-01 10:00:50'),
+ (9, 1, '2024-01-01 10:00:51'), (10, 1, '2024-01-01 10:01:00')
+ """
+
+ // Should find the closest match: ts=10:00:50 (exact match) for >= case
+ qt_asof_many_ge """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_many_matches_left l
+ ASOF LEFT JOIN asof_many_matches_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Should find the closest match: ts=10:00:49 for > case
+ qt_asof_many_gt """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_many_matches_left l
+ ASOF LEFT JOIN asof_many_matches_right r
+ MATCH_CONDITION(l.ts > r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Should find the closest match: ts=10:00:50 for <= case
+ qt_asof_many_le """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_many_matches_left l
+ ASOF LEFT JOIN asof_many_matches_right r
+ MATCH_CONDITION(l.ts <= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Should find the closest match: ts=10:00:51 for < case
+ qt_asof_many_lt """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_many_matches_left l
+ ASOF LEFT JOIN asof_many_matches_right r
+ MATCH_CONDITION(l.ts < r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 14: ASOF JOIN in subquery ====================
+ qt_asof_subquery """
+ SELECT * FROM (
+ SELECT t.trade_id, t.symbol, q.quote_id, q.bid_price
+ FROM asof_trades t
+ ASOF LEFT JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ ) sub
+ WHERE sub.bid_price > 150
+ ORDER BY sub.trade_id
+ """
+
+ // ==================== Test 15: ASOF JOIN with aggregation ====================
+ qt_asof_agg """
+ SELECT t.symbol, COUNT(*) as trade_count, AVG(q.bid_price) as avg_bid
+ FROM asof_trades t
+ ASOF LEFT JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ GROUP BY t.symbol
+ ORDER BY t.symbol
+ """
+
+ // ==================== Test 16: ASOF JOIN with filter on both sides ====================
+ qt_asof_filter """
+ SELECT t.trade_id, t.symbol, t.trade_time,
+ q.quote_id, q.quote_time
+ FROM asof_trades t
+ ASOF LEFT JOIN asof_quotes q
+ MATCH_CONDITION(t.trade_time >= q.quote_time)
+ ON t.symbol = q.symbol
+ WHERE t.price > 150 AND (q.bid_price IS NULL OR q.bid_price > 100)
+ ORDER BY t.trade_id
+ """
+
+ // ==================== Test 17: ASOF INNER JOIN filters out non-matches ====================
+ qt_asof_inner_filter """
+ SELECT l.id, l.grp, l.ts, r.id as rid, r.ts as rts
+ FROM asof_no_match_left l
+ ASOF INNER JOIN asof_no_match_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Complex Test Cases ====================
+
+ // Create tables for complex tests
+ sql """ DROP TABLE IF EXISTS asof_orders """
+ sql """ DROP TABLE IF EXISTS asof_prices """
+ sql """ DROP TABLE IF EXISTS asof_inventory """
+ sql """ DROP TABLE IF EXISTS asof_promotions """
+
+ sql """
+ CREATE TABLE asof_orders (
+ order_id INT,
+ product_id INT,
+ region VARCHAR(10),
+ order_time DATETIME,
+ quantity INT
+ ) DISTRIBUTED BY HASH(order_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_prices (
+ price_id INT,
+ product_id INT,
+ region VARCHAR(10),
+ effective_time DATETIME,
+ price DECIMAL(10,2)
+ ) DISTRIBUTED BY HASH(price_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_inventory (
+ inv_id INT,
+ product_id INT,
+ region VARCHAR(10),
+ snapshot_time DATETIME,
+ stock_level INT
+ ) DISTRIBUTED BY HASH(inv_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_promotions (
+ promo_id INT,
+ product_id INT,
+ start_time DATETIME,
+ discount_pct INT
+ ) DISTRIBUTED BY HASH(promo_id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Insert test data
+ sql """
+ INSERT INTO asof_orders VALUES
+ (1, 100, 'US', '2024-01-01 10:00:00', 5),
+ (2, 100, 'US', '2024-01-01 11:00:00', 3),
+ (3, 100, 'EU', '2024-01-01 10:30:00', 2),
+ (4, 200, 'US', '2024-01-01 10:15:00', 10),
+ (5, 200, 'US', '2024-01-01 12:00:00', 7),
+ (6, 100, 'US', '2024-01-01 09:00:00', 1)
+ """
+
+ sql """
+ INSERT INTO asof_prices VALUES
+ (1, 100, 'US', '2024-01-01 08:00:00', 10.00),
+ (2, 100, 'US', '2024-01-01 09:30:00', 12.00),
+ (3, 100, 'US', '2024-01-01 10:30:00', 11.50),
+ (4, 100, 'EU', '2024-01-01 08:00:00', 15.00),
+ (5, 100, 'EU', '2024-01-01 10:00:00', 14.00),
+ (6, 200, 'US', '2024-01-01 08:00:00', 25.00),
+ (7, 200, 'US', '2024-01-01 11:00:00', 27.00)
+ """
+
+ sql """
+ INSERT INTO asof_inventory VALUES
+ (1, 100, 'US', '2024-01-01 07:00:00', 100),
+ (2, 100, 'US', '2024-01-01 09:00:00', 80),
+ (3, 100, 'US', '2024-01-01 10:30:00', 50),
+ (4, 100, 'EU', '2024-01-01 08:00:00', 200),
+ (5, 200, 'US', '2024-01-01 08:00:00', 500),
+ (6, 200, 'US', '2024-01-01 11:30:00', 400)
+ """
+
+ sql """
+ INSERT INTO asof_promotions VALUES
+ (1, 100, '2024-01-01 09:00:00', 10),
+ (2, 100, '2024-01-01 10:30:00', 15),
+ (3, 200, '2024-01-01 10:00:00', 5)
+ """
+
+ // ==================== Test 18: Multi-level ASOF JOIN (3 tables) ====================
+ // Order -> Price (ASOF) -> Inventory (ASOF)
+ qt_asof_multi_level_3 """
+ SELECT o.order_id, o.product_id, o.order_time, o.quantity,
+ p.price, p.effective_time as price_time,
+ i.stock_level, i.snapshot_time as inv_time
+ FROM asof_orders o
+ ASOF LEFT JOIN asof_prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ASOF LEFT JOIN asof_inventory i
+ MATCH_CONDITION(o.order_time >= i.snapshot_time)
+ ON o.product_id = i.product_id AND o.region = i.region
+ ORDER BY o.order_id
+ """
+
+ // ==================== Test 19: Multi-level ASOF JOIN (4 tables) ====================
+ // Order -> Price -> Inventory -> Promotions
+ qt_asof_multi_level_4 """
+ SELECT o.order_id, o.product_id, o.order_time,
+ p.price,
+ i.stock_level,
+ pr.discount_pct
+ FROM asof_orders o
+ ASOF LEFT JOIN asof_prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ASOF LEFT JOIN asof_inventory i
+ MATCH_CONDITION(o.order_time >= i.snapshot_time)
+ ON o.product_id = i.product_id AND o.region = i.region
+ ASOF LEFT JOIN asof_promotions pr
+ MATCH_CONDITION(o.order_time >= pr.start_time)
+ ON o.product_id = pr.product_id
+ ORDER BY o.order_id
+ """
+
+ // ==================== Test 20: ASOF JOIN mixed with regular JOIN ====================
+ sql """ DROP TABLE IF EXISTS asof_products """
+ sql """
+ CREATE TABLE asof_products (
+ product_id INT,
+ product_name VARCHAR(50),
+ category VARCHAR(20)
+ ) DISTRIBUTED BY HASH(product_id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO asof_products VALUES
+ (100, 'Widget A', 'Electronics'),
+ (200, 'Widget B', 'Electronics'),
+ (300, 'Gadget C', 'Accessories')
+ """
+
+ qt_asof_with_regular_join """
+ SELECT o.order_id, prod.product_name, prod.category,
+ o.order_time, p.price, p.effective_time
+ FROM asof_orders o
+ INNER JOIN asof_products prod ON o.product_id = prod.product_id
+ ASOF LEFT JOIN asof_prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ORDER BY o.order_id
+ """
+
+ // ==================== Test 21: Expression-based equality key ====================
+ sql """ DROP TABLE IF EXISTS asof_join_left """
+ sql """ DROP TABLE IF EXISTS asof_join_right """
+
+ sql """
+ CREATE TABLE asof_join_left (
+ id INT,
+ val1 INT,
+ val2 INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_join_right (
+ id INT,
+ combined_val INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_join_left VALUES
+ (1, 10, 5, '2024-01-01 10:00:00'),
+ (2, 20, 10, '2024-01-01 11:00:00'),
+ (3, 15, 5, '2024-01-01 12:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_join_right VALUES
+ (1, 15, '2024-01-01 09:00:00', 'data_a'),
+ (2, 15, '2024-01-01 09:30:00', 'data_b'),
+ (3, 15, '2024-01-01 11:30:00', 'data_c'),
+ (4, 30, '2024-01-01 10:00:00', 'data_d'),
+ (5, 30, '2024-01-01 10:30:00', 'data_e'),
+ (6, 20, '2024-01-01 11:00:00', 'data_f')
+ """
+
+ // Join on expression: val1 + val2 = combined_val
+ qt_asof_expr_key """
+ SELECT l.id, l.val1, l.val2, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_join_left l
+ ASOF LEFT JOIN asof_join_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.val1 + l.val2 = r.combined_val
+ ORDER BY l.id
+ """
+
+ // ==================== Test 22: String function in equality key ====================
+ sql """ DROP TABLE IF EXISTS asof_str_left """
+ sql """ DROP TABLE IF EXISTS asof_str_right """
+
+ sql """
+ CREATE TABLE asof_str_left (
+ id INT,
+ full_code VARCHAR(20),
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_str_right (
+ id INT,
+ prefix VARCHAR(10),
+ ts DATETIME,
+ value INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_str_left VALUES
+ (1, 'ABC-001', '2024-01-01 10:00:00'),
+ (2, 'ABC-002', '2024-01-01 11:00:00'),
+ (3, 'XYZ-001', '2024-01-01 12:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_str_right VALUES
+ (1, 'ABC', '2024-01-01 09:00:00', 100),
+ (2, 'ABC', '2024-01-01 09:30:00', 110),
+ (3, 'ABC', '2024-01-01 10:30:00', 120),
+ (4, 'XYZ', '2024-01-01 11:00:00', 200),
+ (5, 'XYZ', '2024-01-01 11:30:00', 210)
+ """
+
+ // Join on SUBSTRING expression
+ qt_asof_str_expr """
+ SELECT l.id, l.full_code, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_str_left l
+ ASOF LEFT JOIN asof_str_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON SUBSTRING(l.full_code, 1, 3) = r.prefix
+ ORDER BY l.id
+ """
+
+ // ==================== Test 23: ASOF JOIN with self-join ====================
+ // Note: self-join with inequality in ON clause not supported, use WHERE instead
+ qt_asof_self_join """
+ SELECT t1.order_id as order1, t1.order_time as time1,
+ t2.order_id as order2, t2.order_time as time2
+ FROM asof_orders t1
+ ASOF LEFT JOIN asof_orders t2
+ MATCH_CONDITION(t1.order_time > t2.order_time)
+ ON t1.product_id = t2.product_id AND t1.region = t2.region
+ ORDER BY t1.order_id
+ """
+
+ // ==================== Test 24: ASOF JOIN with date arithmetic in MATCH_CONDITION ====================
+ // Note: MATCH_CONDITION requires direct column comparison, this tests boundary behavior
+ sql """ DROP TABLE IF EXISTS asof_date_arith_left """
+ sql """ DROP TABLE IF EXISTS asof_date_arith_right """
+
+ sql """
+ CREATE TABLE asof_date_arith_left (
+ id INT,
+ grp INT,
+ event_date DATE
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_date_arith_right (
+ id INT,
+ grp INT,
+ ref_date DATE,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_date_arith_left VALUES
+ (1, 1, '2024-01-10'),
+ (2, 1, '2024-01-15'),
+ (3, 2, '2024-01-20')
+ """
+
+ sql """
+ INSERT INTO asof_date_arith_right VALUES
+ (1, 1, '2024-01-05', 'early'),
+ (2, 1, '2024-01-08', 'mid'),
+ (3, 1, '2024-01-12', 'late'),
+ (4, 2, '2024-01-18', 'data_x')
+ """
+
+ qt_asof_date """
+ SELECT l.id, l.event_date, r.id as rid, r.ref_date, r.data
+ FROM asof_date_arith_left l
+ ASOF LEFT JOIN asof_date_arith_right r
+ MATCH_CONDITION(l.event_date >= r.ref_date)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 25: Complex aggregation after multi-level ASOF JOIN ====================
+ qt_asof_complex_agg """
+ SELECT o.product_id, o.region,
+ COUNT(*) as order_count,
+ SUM(o.quantity) as total_qty,
+ AVG(p.price) as avg_price,
+ MIN(i.stock_level) as min_stock
+ FROM asof_orders o
+ ASOF LEFT JOIN asof_prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ASOF LEFT JOIN asof_inventory i
+ MATCH_CONDITION(o.order_time >= i.snapshot_time)
+ ON o.product_id = i.product_id AND o.region = i.region
+ GROUP BY o.product_id, o.region
+ ORDER BY o.product_id, o.region
+ """
+
+ // ==================== Test 26: ASOF JOIN with CASE expression in SELECT ====================
+ qt_asof_case_expr """
+ SELECT o.order_id,
+ CASE
+ WHEN p.price IS NULL THEN 'NO_PRICE'
+ WHEN p.price < 15 THEN 'LOW'
+ WHEN p.price < 25 THEN 'MEDIUM'
+ ELSE 'HIGH'
+ END as price_tier,
+ o.quantity * COALESCE(p.price, 0) as total_value
+ FROM asof_orders o
+ ASOF LEFT JOIN asof_prices p
+ MATCH_CONDITION(o.order_time >= p.effective_time)
+ ON o.product_id = p.product_id AND o.region = p.region
+ ORDER BY o.order_id
+ """
+
+ // ==================== Test 27: Multiple ASOF JOINs with different directions ====================
+ qt_asof_mixed_directions """
+ SELECT o.order_id, o.order_time,
+ p_before.price as price_before,
+ p_before.effective_time as time_before,
+ p_after.price as price_after,
+ p_after.effective_time as time_after
+ FROM asof_orders o
+ ASOF LEFT JOIN asof_prices p_before
+ MATCH_CONDITION(o.order_time >= p_before.effective_time)
+ ON o.product_id = p_before.product_id AND o.region = p_before.region
+ ASOF LEFT JOIN asof_prices p_after
+ MATCH_CONDITION(o.order_time <= p_after.effective_time)
+ ON o.product_id = p_after.product_id AND o.region = p_after.region
+ WHERE o.product_id = 100 AND o.region = 'US'
+ ORDER BY o.order_id
+ """
+
+ // ==================== Corner Case Tests ====================
+
+ // ==================== Test 28: Duplicate timestamps on build side ====================
+ // When multiple build rows have the exact same timestamp, behavior should be deterministic
+ sql """ DROP TABLE IF EXISTS asof_dup_ts_left """
+ sql """ DROP TABLE IF EXISTS asof_dup_ts_right """
+
+ sql """
+ CREATE TABLE asof_dup_ts_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_dup_ts_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ value INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_dup_ts_left VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 1, '2024-01-01 10:00:30')
+ """
+
+ // Multiple rows with SAME timestamp
+ sql """
+ INSERT INTO asof_dup_ts_right VALUES
+ (1, 1, '2024-01-01 09:30:00', 100),
+ (2, 1, '2024-01-01 09:30:00', 200),
+ (3, 1, '2024-01-01 09:30:00', 300),
+ (4, 1, '2024-01-01 10:00:00', 400),
+ (5, 1, '2024-01-01 10:00:00', 500)
+ """
+
+ // For >= with duplicate timestamps, should return one of the matching rows
+ // When multiple build rows have same ts, any one is valid - use validation instead of qt_sql
+ def dup_ts_result = sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_dup_ts_left l
+ ASOF LEFT JOIN asof_dup_ts_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+ // Validate: each row should match one of the valid build rows with max ts <= probe ts
+ // For l.id=1 (ts=10:00): valid matches are id=4 or id=5 (both ts=10:00)
+ // For l.id=2 (ts=10:00:30): valid matches are id=4 or id=5 (both ts=10:00, which is max <= 10:00:30)
+ assertEquals(2, dup_ts_result.size())
+ // Row 0: l.id=1, rid should be 4 or 5, rts should be 10:00, value should be 400 or 500
+ assertEquals(1, dup_ts_result[0][0])
+ assertTrue(dup_ts_result[0][2] == 4 || dup_ts_result[0][2] == 5, "rid should be 4 or 5, got: ${dup_ts_result[0][2]}")
+ assertTrue(dup_ts_result[0][4] == 400 || dup_ts_result[0][4] == 500, "value should be 400 or 500")
+ // Row 1: l.id=2, rid should be 4 or 5
+ assertEquals(2, dup_ts_result[1][0])
+ assertTrue(dup_ts_result[1][2] == 4 || dup_ts_result[1][2] == 5, "rid should be 4 or 5, got: ${dup_ts_result[1][2]}")
+ assertTrue(dup_ts_result[1][4] == 400 || dup_ts_result[1][4] == 500, "value should be 400 or 500")
+
+ // ==================== Test 29: Exact boundary with strict inequality ====================
+ sql """ DROP TABLE IF EXISTS asof_boundary_left """
+ sql """ DROP TABLE IF EXISTS asof_boundary_right """
+
+ sql """
+ CREATE TABLE asof_boundary_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_boundary_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_boundary_left VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 1, '2024-01-01 10:00:01'),
+ (3, 1, '2024-01-01 09:59:59')
+ """
+
+ sql """
+ INSERT INTO asof_boundary_right VALUES
+ (1, 1, '2024-01-01 10:00:00', 'exact'),
+ (2, 1, '2024-01-01 09:00:00', 'before'),
+ (3, 1, '2024-01-01 11:00:00', 'after')
+ """
+
+ // Test > (strict greater): probe.ts=10:00:00 should NOT match build.ts=10:00:00
+ qt_asof_boundary_gt """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_boundary_left l
+ ASOF LEFT JOIN asof_boundary_right r
+ MATCH_CONDITION(l.ts > r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Test < (strict less): probe.ts=10:00:00 should NOT match build.ts=10:00:00
+ qt_asof_boundary_lt """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_boundary_left l
+ ASOF LEFT JOIN asof_boundary_right r
+ MATCH_CONDITION(l.ts < r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Test >= : probe.ts=10:00:00 SHOULD match build.ts=10:00:00
+ qt_asof_boundary_ge """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_boundary_left l
+ ASOF LEFT JOIN asof_boundary_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Test <= : probe.ts=10:00:00 SHOULD match build.ts=10:00:00
+ qt_asof_boundary_le """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_boundary_left l
+ ASOF LEFT JOIN asof_boundary_right r
+ MATCH_CONDITION(l.ts <= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 30: All build rows filtered out ====================
+ sql """ DROP TABLE IF EXISTS asof_all_filtered_left """
+ sql """ DROP TABLE IF EXISTS asof_all_filtered_right """
+
+ sql """
+ CREATE TABLE asof_all_filtered_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_all_filtered_right (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_all_filtered_left VALUES
+ (1, 1, '2024-01-01 08:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_all_filtered_right VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 1, '2024-01-01 11:00:00'),
+ (3, 1, '2024-01-01 12:00:00')
+ """
+
+ // All build rows have ts > probe.ts, so >= should find nothing
+ qt_asof_all_filtered """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_all_filtered_left l
+ ASOF LEFT JOIN asof_all_filtered_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 31: Multiple groups with different match patterns ====================
+ sql """ DROP TABLE IF EXISTS asof_multi_grp_left """
+ sql """ DROP TABLE IF EXISTS asof_multi_grp_right """
+
+ sql """
+ CREATE TABLE asof_multi_grp_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_multi_grp_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ value INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_multi_grp_left VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 2, '2024-01-01 10:00:00'),
+ (3, 3, '2024-01-01 10:00:00'),
+ (4, 4, '2024-01-01 10:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_multi_grp_right VALUES
+ (1, 1, '2024-01-01 09:00:00', 100),
+ (2, 1, '2024-01-01 09:30:00', 110),
+ (3, 1, '2024-01-01 09:45:00', 120),
+ (4, 2, '2024-01-01 11:00:00', 200),
+ (5, 3, '2024-01-01 10:00:00', 300)
+ """
+ // grp=1: has matches before 10:00:00
+ // grp=2: only has match after 10:00:00 (no match for >=)
+ // grp=3: exact match at 10:00:00
+ // grp=4: no build rows at all
+
+ qt_asof_multi_grp """
+ SELECT l.id, l.grp, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_multi_grp_left l
+ ASOF LEFT JOIN asof_multi_grp_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 32: Microsecond precision ====================
+ sql """ DROP TABLE IF EXISTS asof_precision_left """
+ sql """ DROP TABLE IF EXISTS asof_precision_right """
+
+ sql """
+ CREATE TABLE asof_precision_left (
+ id INT,
+ grp INT,
+ ts DATETIME(6)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_precision_right (
+ id INT,
+ grp INT,
+ ts DATETIME(6),
+ value INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_precision_left VALUES
+ (1, 1, '2024-01-01 10:00:00.000500'),
+ (2, 1, '2024-01-01 10:00:00.001000')
+ """
+
+ sql """
+ INSERT INTO asof_precision_right VALUES
+ (1, 1, '2024-01-01 10:00:00.000000', 100),
+ (2, 1, '2024-01-01 10:00:00.000400', 200),
+ (3, 1, '2024-01-01 10:00:00.000600', 300),
+ (4, 1, '2024-01-01 10:00:00.001000', 400)
+ """
+
+ qt_asof_precision """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= l.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+ exception "MATCH_CONDITION must specify column from both table"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts - r.ts >= 1)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+ exception "only allow date, datetime and timestamptz in MATCH_CONDITION"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp or l.id > r.id
+ ORDER BY l.id
+ """
+ exception "ASOF JOIN's ON clause must be one or more EQUAL(=) conjuncts"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+ exception "only ASOF JOIN support MATCH_CONDITION"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ORDER BY l.id
+ """
+ exception "ASOF JOIN must have on or using clause"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts == r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+ exception "ASOF JOIN's MATCH_CONDITION must be <, <=, >, >="
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+ exception "ASOF JOIN must specify MATCH_CONDITION"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = 1
+ ORDER BY l.id
+ """
+ exception "ASOF join's hash conjuncts must be in form of"
+ }
+
+ test {
+ sql """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.value
+ FROM asof_precision_left l
+ ASOF LEFT JOIN asof_precision_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp and l.grp = 1
+ ORDER BY l.id
+ """
+ exception "ASOF join's hash conjuncts must be in form of"
+ }
+
+ // ==================== Test 33-40: Comprehensive ASOF JOIN Coverage ====================
+ // Cover: LEFT x INNER/OUTER x operators(>=,<=,>,<) x SlotRef/Expression
+ // Use explain + contains to verify plan type before execution
+
+ // ---------- Setup: Tables for ASOF JOIN tests ----------
+ sql """ DROP TABLE IF EXISTS asof_small """
+ sql """ DROP TABLE IF EXISTS asof_large """
+
+ // Small table (4 rows)
+ sql """
+ CREATE TABLE asof_small (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Large table (20 rows) - will become probe side
+ sql """
+ CREATE TABLE asof_large (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ val INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_small VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 1, '2024-01-01 12:00:00'),
+ (3, 2, '2024-01-01 11:00:00'),
+ (4, 4, '2024-01-01 10:00:00')
+ """
+
+ // Insert 20 rows
+ sql """
+ INSERT INTO asof_large VALUES
+ (1, 1, '2024-01-01 08:00:00', 100),
+ (2, 1, '2024-01-01 09:00:00', 110),
+ (3, 1, '2024-01-01 10:00:00', 120),
+ (4, 1, '2024-01-01 11:00:00', 130),
+ (5, 1, '2024-01-01 12:00:00', 140),
+ (6, 1, '2024-01-01 13:00:00', 150),
+ (7, 2, '2024-01-01 09:00:00', 200),
+ (8, 2, '2024-01-01 10:00:00', 210),
+ (9, 2, '2024-01-01 11:00:00', 220),
+ (10, 2, '2024-01-01 12:00:00', 230),
+ (11, 1, '2024-01-01 09:30:00', 115),
+ (12, 1, '2024-01-01 10:30:00', 125),
+ (13, 1, '2024-01-01 11:30:00', 135),
+ (14, 2, '2024-01-01 09:30:00', 205),
+ (15, 2, '2024-01-01 10:30:00', 215),
+ (16, 2, '2024-01-01 11:30:00', 225),
+ (17, 1, '2024-01-01 14:00:00', 160),
+ (18, 2, '2024-01-01 13:00:00', 240),
+ (19, 3, '2024-01-01 10:00:00', 300),
+ (20, 3, '2024-01-01 11:00:00', 310)
+ """
+
+ sql """ ANALYZE TABLE asof_small WITH SYNC """
+ sql """ ANALYZE TABLE asof_large WITH SYNC """
+
+ // ---------- Test 33: ASOF LEFT JOIN (verify plan) ----------
+ // Swap table order: large table on left forces LEFT plan
+ explain {
+ sql """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts >= s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+ contains "ASOF_LEFT_INNER_JOIN"
+ }
+
+ qt_asof_left_inner_ge """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts >= s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+
+ // ---------- Test 39: ASOF LEFT OUTER JOIN with unmatched rows ----------
+ explain {
+ sql """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF JOIN asof_small s
+ MATCH_CONDITION(l.ts >= s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+ contains "ASOF_LEFT_OUTER_JOIN"
+ }
+
+ qt_asof_left_outer_ge """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF JOIN asof_small s
+ MATCH_CONDITION(l.ts >= s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+
+ // ---------- Test 40: Expression-based MATCH_CONDITION (forces linear scan) ----------
+ // Use expression instead of simple SlotRef to verify expression path
+ qt_asof_expr_condition """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts >= DATE_SUB(s.ts, INTERVAL 0 SECOND))
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+
+ // ---------- Test 38: Large group to trigger binary search (LEFT JOIN) ----------
+ // Binary search is triggered when group_size > 8 and uses simple SlotRef
+ // Use asof_large (20 rows) as left table and asof_small (4 rows) as right table
+ // This forces LEFT plan and tests binary search optimization
+
+ explain {
+ sql """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts >= s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+ contains "ASOF_LEFT_INNER_JOIN"
+ }
+
+ // This triggers binary search path when group has > 8 candidates
+ qt_asof_bsearch_ge """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts >= s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+
+ // Test binary search with <= operator
+ qt_asof_bsearch_le """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts <= s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+
+ // Test binary search with strict > operator
+ qt_asof_bsearch_gt """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts > s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+
+ // Test binary search with strict < operator
+ qt_asof_bsearch_lt """
+ SELECT l.id, l.grp, l.ts as l_ts, s.ts as s_ts, s.id as sid
+ FROM asof_large l
+ ASOF INNER JOIN asof_small s
+ MATCH_CONDITION(l.ts < s.ts)
+ ON l.grp = s.grp
+ ORDER BY l.id
+ """
+
+ // ---------- Test 43: Multiple groups with different match scenarios ----------
+ sql """ DROP TABLE IF EXISTS asof_multi_grp_left2 """
+ sql """ DROP TABLE IF EXISTS asof_multi_grp_right2 """
+
+ sql """
+ CREATE TABLE asof_multi_grp_left2 (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_multi_grp_right2 (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ val INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Insert 10 rows in left table to force LEFT plan (larger than right table's 6 rows)
+ sql """
+ INSERT INTO asof_multi_grp_left2 VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 2, '2024-01-01 10:00:00'),
+ (3, 3, '2024-01-01 10:00:00'),
+ (4, 4, '2024-01-01 10:00:00'),
+ (5, 1, '2024-01-01 11:00:00'),
+ (6, 2, '2024-01-01 09:00:00'),
+ (7, 3, '2024-01-01 11:00:00'),
+ (8, 1, '2024-01-01 09:00:00'),
+ (9, 2, '2024-01-01 11:00:00'),
+ (10, 3, '2024-01-01 08:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_multi_grp_right2 VALUES
+ (1, 1, '2024-01-01 09:00:00', 100),
+ (2, 1, '2024-01-01 10:00:00', 110),
+ (3, 1, '2024-01-01 11:00:00', 120),
+ (4, 2, '2024-01-01 11:00:00', 200),
+ (5, 3, '2024-01-01 09:00:00', 300),
+ (6, 3, '2024-01-01 10:00:00', 310)
+ """
+
+ sql """ ANALYZE TABLE asof_multi_grp_left2 WITH SYNC """
+ sql """ ANALYZE TABLE asof_multi_grp_right2 WITH SYNC """
+
+ // Test multiple groups with LEFT OUTER JOIN
+ // Verify plan type first
+ explain {
+ sql """
+ SELECT l.id, l.grp, l.ts as l_ts, r.ts as r_ts, r.val
+ FROM asof_multi_grp_left2 l
+ ASOF LEFT JOIN asof_multi_grp_right2 r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+ contains "ASOF_LEFT_OUTER_JOIN"
+ }
+
+ // Expected results:
+ // grp=1: id=1 ts=10:00 -> r.ts<=10:00 -> 10:00(val=110); id=5 ts=11:00 -> 11:00(val=120); id=8 ts=09:00 -> 09:00(val=100)
+ // grp=2: id=2 ts=10:00 -> r.ts<=10:00? r only has 11:00 -> NULL; id=6 ts=09:00 -> NULL; id=9 ts=11:00 -> 11:00(val=200)
+ // grp=3: id=3 ts=10:00 -> 10:00(val=310); id=7 ts=11:00 -> 10:00(val=310); id=10 ts=08:00 -> NULL
+ // grp=4: id=4 -> no r row -> NULL
+ qt_asof_multi_grp_outer """
+ SELECT l.id, l.grp, l.ts as l_ts, r.ts as r_ts, r.val
+ FROM asof_multi_grp_left2 l
+ ASOF LEFT JOIN asof_multi_grp_right2 r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 44: TIMESTAMPTZ type support ====================
+ sql """ DROP TABLE IF EXISTS asof_timestamptz_left """
+ sql """ DROP TABLE IF EXISTS asof_timestamptz_right """
+
+ sql """
+ CREATE TABLE asof_timestamptz_left (
+ id INT,
+ grp INT,
+ ts TIMESTAMPTZ
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_timestamptz_right (
+ id INT,
+ grp INT,
+ ts TIMESTAMPTZ,
+ val INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_timestamptz_left VALUES
+ (1, 1, '2024-01-01 10:00:00+08:00'),
+ (2, 1, '2024-01-01 12:00:00+08:00')
+ """
+
+ sql """
+ INSERT INTO asof_timestamptz_right VALUES
+ (1, 1, '2024-01-01 09:00:00+08:00', 100),
+ (2, 1, '2024-01-01 10:00:00+08:00', 200),
+ (3, 1, '2024-01-01 11:00:00+08:00', 300)
+ """
+
+ qt_asof_timestamptz """
+ SELECT l.id, l.ts as l_ts, r.id as rid, r.ts as r_ts, r.val
+ FROM asof_timestamptz_left l
+ ASOF LEFT JOIN asof_timestamptz_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+}
diff --git a/regression-test/suites/nereids_p0/join/asof/test_asof_join_build_expr_edge.groovy b/regression-test/suites/nereids_p0/join/asof/test_asof_join_build_expr_edge.groovy
new file mode 100644
index 00000000000..d156ec523df
--- /dev/null
+++ b/regression-test/suites/nereids_p0/join/asof/test_asof_join_build_expr_edge.groovy
@@ -0,0 +1,401 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Edge-case tests for ASOF JOIN build-side expression computation.
+// Focus: build_asof_index() correctly evaluates expressions when the build side
+// comes from complex plan structures (CTE, subquery, multi-join, chaining, aggregation).
+// The build-side expression is prepared against the intermediate row descriptor
+// and executed on a dummy-probe + build block; these tests verify that slot ID
+// mapping works correctly across various plan topologies.
+
+suite("test_asof_join_build_expr_edge", "nereids_p0") {
+
+ // ====================== Base table setup ======================
+ sql "DROP TABLE IF EXISTS abee_probe"
+ sql "DROP TABLE IF EXISTS abee_build"
+ sql "DROP TABLE IF EXISTS abee_dim"
+ sql "DROP TABLE IF EXISTS abee_build2"
+ sql "DROP TABLE IF EXISTS abee_events"
+
+ sql """
+ CREATE TABLE abee_probe (
+ id INT,
+ grp INT,
+ grp2 INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE abee_build (
+ id INT,
+ grp INT,
+ grp2 INT,
+ ts DATETIME,
+ val INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE abee_dim (
+ id INT,
+ grp INT,
+ name VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE abee_build2 (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE abee_events (
+ id INT,
+ grp INT,
+ category VARCHAR(10),
+ ts DATETIME,
+ amount INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // -- Probe data --
+ sql """
+ INSERT INTO abee_probe VALUES
+ (1, 1, 10, '2024-01-01 10:00:00'),
+ (2, 1, 10, '2024-01-01 12:00:00'),
+ (3, 2, 20, '2024-01-01 11:00:00'),
+ (4, 2, 20, '2024-01-01 14:00:00'),
+ (5, 3, 30, '2024-01-01 10:00:00')
+ """
+
+ // -- Build data (4 rows per grp=1, 4 per grp=2, 2 per grp=3) --
+ sql """
+ INSERT INTO abee_build VALUES
+ (1, 1, 10, '2024-01-01 08:00:00', 100),
+ (2, 1, 10, '2024-01-01 09:00:00', 110),
+ (3, 1, 10, '2024-01-01 10:00:00', 120),
+ (4, 1, 10, '2024-01-01 11:00:00', 130),
+ (5, 2, 20, '2024-01-01 09:00:00', 200),
+ (6, 2, 20, '2024-01-01 10:00:00', 210),
+ (7, 2, 20, '2024-01-01 12:00:00', 220),
+ (8, 2, 20, '2024-01-01 13:00:00', 230),
+ (9, 3, 30, '2024-01-01 09:00:00', 300),
+ (10, 3, 30, '2024-01-01 09:30:00', 310)
+ """
+
+ // -- Dimension data --
+ sql """
+ INSERT INTO abee_dim VALUES
+ (1, 1, 'alpha'),
+ (2, 2, 'beta'),
+ (3, 3, 'gamma')
+ """
+
+ // -- Second build table for chained-join test --
+ sql """
+ INSERT INTO abee_build2 VALUES
+ (1, 1, '2024-01-01 07:00:00', 'b2_1'),
+ (2, 1, '2024-01-01 09:00:00', 'b2_2'),
+ (3, 2, '2024-01-01 08:00:00', 'b2_3'),
+ (4, 2, '2024-01-01 11:00:00', 'b2_4'),
+ (5, 3, '2024-01-01 08:30:00', 'b2_5')
+ """
+
+ // -- Events data for aggregation test --
+ sql """
+ INSERT INTO abee_events VALUES
+ (1, 1, 'A', '2024-01-01 08:00:00', 10),
+ (2, 1, 'A', '2024-01-01 08:30:00', 20),
+ (3, 1, 'B', '2024-01-01 09:00:00', 30),
+ (4, 1, 'B', '2024-01-01 09:30:00', 40),
+ (5, 2, 'A', '2024-01-01 10:00:00', 50),
+ (6, 2, 'A', '2024-01-01 10:30:00', 60),
+ (7, 2, 'B', '2024-01-01 11:00:00', 70),
+ (8, 2, 'B', '2024-01-01 11:30:00', 80)
+ """
+
+ // =====================================================================
+ // Test 1: CTE as both probe and build with build-side expression
+ // =====================================================================
+ qt_abee_cte_expr """
+ WITH cte_p AS (SELECT id, grp, ts FROM abee_probe),
+ cte_b AS (SELECT id, grp, ts, val FROM abee_build)
+ SELECT l.id, r.id as rid, r.val
+ FROM cte_p l
+ ASOF LEFT JOIN cte_b r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 2: Inline view (subquery) as build side with expression
+ // =====================================================================
+ qt_abee_inline_view """
+ SELECT l.id, r.rid, r.val
+ FROM abee_probe l
+ ASOF LEFT JOIN (
+ SELECT id AS rid, grp, ts AS rts, val FROM abee_build
+ ) r
+ MATCH_CONDITION(l.ts >= r.rts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 3: Build side from regular JOIN result + expression
+ // Build child = abee_build INNER JOIN abee_dim (adds name column)
+ // =====================================================================
+ qt_abee_join_build """
+ SELECT l.id, r.rid, r.val, r.name
+ FROM abee_probe l
+ ASOF LEFT JOIN (
+ SELECT b.id AS rid, b.grp, b.ts, b.val, d.name
+ FROM abee_build b INNER JOIN abee_dim d ON b.grp = d.grp
+ ) r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 4: Chained ASOF JOINs — second join has build-side expression
+ // =====================================================================
+ qt_abee_chained """
+ SELECT l.id, r1.id AS r1id, r1.val, r2.id AS r2id, r2.data
+ FROM abee_probe l
+ ASOF LEFT JOIN abee_build r1
+ MATCH_CONDITION(l.ts >= r1.ts)
+ ON l.grp = r1.grp
+ ASOF LEFT JOIN abee_build2 r2
+ MATCH_CONDITION(l.ts >= r2.ts + INTERVAL 1 HOUR)
+ ON l.grp = r2.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 5: Multiple equality keys with expression
+ // ON l.grp = r.grp AND l.grp2 = r.grp2
+ // Tests hash collision handling with compound keys.
+ // =====================================================================
+ qt_abee_compound_keys """
+ SELECT l.id, r.id AS rid, r.val
+ FROM abee_probe l
+ ASOF LEFT JOIN abee_build r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp AND l.grp2 = r.grp2
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 6: Self-ASOF-JOIN via CTE with expression
+ // Same table as both probe and build. Tests CTE shared-scan +
+ // intermediate row desc when both sides have identical tuples.
+ // =====================================================================
+ qt_abee_self_join """
+ WITH data AS (SELECT id, grp, ts, val FROM abee_build)
+ SELECT l.id, r.id AS rid, r.val
+ FROM data l
+ ASOF LEFT JOIN data r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 7: Cross-precision DATETIME with expression
+ // Probe: DATETIMEV2(3), Build: DATETIMEV2(6).
+ // FE inserts implicit CAST; tests that expression evaluation
+ // on build side handles type promotion correctly.
+ // =====================================================================
+ sql "DROP TABLE IF EXISTS abee_ts3"
+ sql "DROP TABLE IF EXISTS abee_ts6"
+
+ sql """
+ CREATE TABLE abee_ts3 (
+ id INT, grp INT, ts DATETIMEV2(3)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+ sql """
+ CREATE TABLE abee_ts6 (
+ id INT, grp INT, ts DATETIMEV2(6), val INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO abee_ts3 VALUES
+ (1, 1, '2024-01-01 10:00:00.123'),
+ (2, 1, '2024-01-01 12:00:00.456')
+ """
+ sql """
+ INSERT INTO abee_ts6 VALUES
+ (1, 1, '2024-01-01 08:00:00.000001', 100),
+ (2, 1, '2024-01-01 09:00:00.123000', 110),
+ (3, 1, '2024-01-01 10:00:00.123000', 120),
+ (4, 1, '2024-01-01 11:00:00.456789', 130)
+ """
+
+ qt_abee_cross_precision """
+ SELECT l.id, r.id AS rid, r.val
+ FROM abee_ts3 l
+ ASOF LEFT JOIN abee_ts6 r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 8: Aggregation subquery as build side with expression
+ // =====================================================================
+ qt_abee_agg_build """
+ SELECT l.id, r.category, r.total
+ FROM abee_probe l
+ ASOF LEFT JOIN (
+ SELECT grp, category, MAX(ts) AS max_ts, SUM(amount) AS total
+ FROM abee_events
+ GROUP BY grp, category
+ ) r
+ MATCH_CONDITION(l.ts >= r.max_ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 9: WHERE clause filtering after ASOF + expression
+ // =====================================================================
+ qt_abee_where_filter """
+ SELECT l.id, r.id AS rid, r.val
+ FROM abee_probe l
+ ASOF LEFT JOIN abee_build r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ WHERE r.val > 150 OR r.val IS NULL
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 10: Nested subquery (2-level deep) as build side
+ // Inner filter: val > 100 → removes id=1(100)
+ // =====================================================================
+ qt_abee_nested_subquery """
+ SELECT l.id, r.rid, r.val
+ FROM abee_probe l
+ ASOF LEFT JOIN (
+ SELECT rid, grp, rts, val FROM (
+ SELECT id AS rid, grp, ts AS rts, val
+ FROM abee_build
+ WHERE val > 100
+ ) inner_q
+ ) r
+ MATCH_CONDITION(l.ts >= r.rts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Test 11: ASOF INNER JOIN with CTE + expression (no NULL padding)
+ // Unmatched probe rows should be dropped entirely.
+ // =====================================================================
+ qt_abee_inner_cte """
+ WITH data AS (SELECT id, grp, ts, val FROM abee_build)
+ SELECT l.id, r.id AS rid, r.val
+ FROM data l
+ ASOF INNER JOIN data r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // =====================================================================
+ // Cross-validation: compare ASOF JOIN result with equivalent
+ // regular LEFT JOIN + ROW_NUMBER to ensure semantic correctness.
+ // Uses Test 1 data (abee_probe ASOF LEFT JOIN abee_build with +1h).
+ // =====================================================================
+ def asof_result = sql """
+ SELECT l.id, r.id AS rid, r.val
+ FROM abee_probe l
+ ASOF LEFT JOIN abee_build r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ def regular_result = sql """
+ SELECT id, rid, val FROM (
+ SELECT l.id, r.id AS rid, r.val,
+ ROW_NUMBER() OVER (PARTITION BY l.id ORDER BY r.ts DESC) AS rn
+ FROM abee_probe l
+ LEFT JOIN abee_build r
+ ON l.grp = r.grp AND l.ts >= r.ts + INTERVAL 1 HOUR
+ ) t
+ WHERE rn = 1
+ ORDER BY id
+ """
+
+ assertEquals(asof_result.size(), regular_result.size())
+ for (int i = 0; i < asof_result.size(); i++) {
+ assertEquals(asof_result[i][0], regular_result[i][0],
+ "Row ${i} id mismatch: ASOF=${asof_result[i][0]} vs Regular=${regular_result[i][0]}")
+ assertEquals(asof_result[i][1], regular_result[i][1],
+ "Row ${i} rid mismatch: ASOF=${asof_result[i][1]} vs Regular=${regular_result[i][1]}")
+ assertEquals(asof_result[i][2], regular_result[i][2],
+ "Row ${i} val mismatch: ASOF=${asof_result[i][2]} vs Regular=${regular_result[i][2]}")
+ }
+
+ // Cross-validate chained join (Test 4) second ASOF result
+ def chain_asof = sql """
+ SELECT l.id, r2.id AS r2id, r2.data
+ FROM abee_probe l
+ ASOF LEFT JOIN abee_build r1
+ MATCH_CONDITION(l.ts >= r1.ts) ON l.grp = r1.grp
+ ASOF LEFT JOIN abee_build2 r2
+ MATCH_CONDITION(l.ts >= r2.ts + INTERVAL 1 HOUR) ON l.grp = r2.grp
+ ORDER BY l.id
+ """
+
+ def chain_regular = sql """
+ SELECT id, r2id, data FROM (
+ SELECT l.id, r2.id AS r2id, r2.data,
+ ROW_NUMBER() OVER (PARTITION BY l.id ORDER BY r2.ts DESC) AS rn
+ FROM abee_probe l
+ LEFT JOIN abee_build2 r2
+ ON l.grp = r2.grp AND l.ts >= r2.ts + INTERVAL 1 HOUR
+ ) t
+ WHERE rn = 1
+ ORDER BY id
+ """
+
+ assertEquals(chain_asof.size(), chain_regular.size())
+ for (int i = 0; i < chain_asof.size(); i++) {
+ assertEquals(chain_asof[i][1], chain_regular[i][1],
+ "Chain row ${i} r2id mismatch: ASOF=${chain_asof[i][1]} vs Regular=${chain_regular[i][1]}")
+ assertEquals(chain_asof[i][2], chain_regular[i][2],
+ "Chain row ${i} data mismatch")
+ }
+}
diff --git a/regression-test/suites/nereids_p0/join/asof/test_asof_join_explicit.groovy b/regression-test/suites/nereids_p0/join/asof/test_asof_join_explicit.groovy
new file mode 100644
index 00000000000..4d4101e74c7
--- /dev/null
+++ b/regression-test/suites/nereids_p0/join/asof/test_asof_join_explicit.groovy
@@ -0,0 +1,369 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_asof_join_explicit") {
+ // ===================================================================================
+ // This test suite verifies ASOF JOIN results are correct with both
+ // disable_join_reorder=true and disable_join_reorder=false settings.
+ // ===================================================================================
+
+ // ==================== Setup Tables ====================
+ // Small table (4 rows) vs Large table (20 rows) - 5x difference for CBO to swap
+ sql "DROP TABLE IF EXISTS asof_explicit_small"
+ sql "DROP TABLE IF EXISTS asof_explicit_large"
+
+ sql """
+ CREATE TABLE asof_explicit_small (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_explicit_large (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ val INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Small table: 4 rows
+ sql """
+ INSERT INTO asof_explicit_small VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 1, '2024-01-01 12:00:00'),
+ (3, 2, '2024-01-01 11:00:00'),
+ (4, 3, '2024-01-01 10:00:00')
+ """
+
+ // Large table: 20 rows (5x difference)
+ sql """
+ INSERT INTO asof_explicit_large VALUES
+ (1, 1, '2024-01-01 08:00:00', 100),
+ (2, 1, '2024-01-01 09:00:00', 110),
+ (3, 1, '2024-01-01 10:00:00', 120),
+ (4, 1, '2024-01-01 11:00:00', 130),
+ (5, 1, '2024-01-01 12:00:00', 140),
+ (6, 1, '2024-01-01 13:00:00', 150),
+ (7, 2, '2024-01-01 09:00:00', 200),
+ (8, 2, '2024-01-01 10:00:00', 210),
+ (9, 2, '2024-01-01 11:00:00', 220),
+ (10, 2, '2024-01-01 12:00:00', 230),
+ (11, 1, '2024-01-01 09:30:00', 115),
+ (12, 1, '2024-01-01 10:30:00', 125),
+ (13, 1, '2024-01-01 11:30:00', 135),
+ (14, 2, '2024-01-01 09:30:00', 205),
+ (15, 2, '2024-01-01 10:30:00', 215),
+ (16, 2, '2024-01-01 11:30:00', 225),
+ (17, 1, '2024-01-01 14:00:00', 160),
+ (18, 2, '2024-01-01 13:00:00', 240),
+ (19, 3, '2024-01-01 09:00:00', 300),
+ (20, 3, '2024-01-01 11:00:00', 310)
+ """
+
+ // CRITICAL: Analyze tables for CBO to make correct decisions
+ sql "ANALYZE TABLE asof_explicit_small WITH SYNC"
+ sql "ANALYZE TABLE asof_explicit_large WITH SYNC"
+
+ // ===================================================================================
+ // Test 1: ASOF LEFT OUTER JOIN with >= operator
+ // ===================================================================================
+
+ // 1a. Verify LEFT JOIN plan with reorder disabled
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts >= l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_OUTER_JOIN"
+ }
+
+ qt_left_outer_ge """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts >= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // 1b. Verify results with reorder enabled
+ sql "SET disable_join_reorder = false"
+ qt_right_outer_ge """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts >= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // ===================================================================================
+ // Test 2: ASOF INNER JOIN with >= operator
+ // ===================================================================================
+
+ // 2a. Verify LEFT INNER plan with reorder disabled
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts >= l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_INNER_JOIN"
+ }
+
+ qt_left_inner_ge """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts >= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // 2b. Verify results with reorder enabled
+ sql "SET disable_join_reorder = false"
+ qt_right_inner_ge """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts >= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // ===================================================================================
+ // Test 3: ASOF LEFT OUTER JOIN with <= operator
+ // ===================================================================================
+
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts <= l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_OUTER_JOIN"
+ }
+
+ qt_left_outer_le """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts <= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ sql "SET disable_join_reorder = false"
+ qt_right_outer_le """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts <= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // ===================================================================================
+ // Test 4: ASOF LEFT OUTER JOIN with > operator (strict)
+ // ===================================================================================
+
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts > l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_OUTER_JOIN"
+ }
+
+ qt_left_outer_gt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts > l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ sql "SET disable_join_reorder = false"
+ qt_right_outer_gt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts > l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // ===================================================================================
+ // Test 5: ASOF LEFT OUTER JOIN with < operator (strict)
+ // ===================================================================================
+
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts < l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_OUTER_JOIN"
+ }
+
+ qt_left_outer_lt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts < l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ sql "SET disable_join_reorder = false"
+ qt_right_outer_lt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF LEFT JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts < l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // ===================================================================================
+ // Test 6: ASOF INNER JOIN with <= operator
+ // ===================================================================================
+
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts <= l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_INNER_JOIN"
+ }
+
+ qt_left_inner_le """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts <= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ sql "SET disable_join_reorder = false"
+ qt_right_inner_le """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts <= l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // ===================================================================================
+ // Test 7: ASOF INNER JOIN with > operator (strict)
+ // ===================================================================================
+
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts > l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_INNER_JOIN"
+ }
+
+ qt_left_inner_gt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts > l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ sql "SET disable_join_reorder = false"
+ qt_right_inner_gt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts > l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ // ===================================================================================
+ // Test 8: ASOF INNER JOIN with < operator (strict)
+ // ===================================================================================
+
+ sql "SET disable_join_reorder = true"
+ explain {
+ sql """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts < l.ts)
+ ON s.grp = l.grp
+ """
+ contains "ASOF_LEFT_INNER_JOIN"
+ }
+
+ qt_left_inner_lt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts < l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+
+ sql "SET disable_join_reorder = false"
+ qt_right_inner_lt """
+ SELECT s.id, s.grp, s.ts as s_ts, l.ts as l_ts, l.val
+ FROM asof_explicit_small s
+ ASOF INNER JOIN asof_explicit_large l
+ MATCH_CONDITION(s.ts < l.ts)
+ ON s.grp = l.grp
+ ORDER BY s.id
+ """
+}
diff --git a/regression-test/suites/nereids_p0/join/asof/test_asof_join_expr.groovy b/regression-test/suites/nereids_p0/join/asof/test_asof_join_expr.groovy
new file mode 100644
index 00000000000..36ac8371689
--- /dev/null
+++ b/regression-test/suites/nereids_p0/join/asof/test_asof_join_expr.groovy
@@ -0,0 +1,477 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Test suite for ASOF JOIN with datetime expressions in MATCH_CONDITION
+// FE allows expressions as long as both sides are date-like types
+
+suite("test_asof_join_expr", "nereids_p0") {
+
+ sql "DROP TABLE IF EXISTS asof_expr_left"
+ sql "DROP TABLE IF EXISTS asof_expr_right"
+
+ sql """
+ CREATE TABLE asof_expr_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_expr_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_expr_left VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 1, '2024-01-01 11:00:00'),
+ (3, 1, '2024-01-01 12:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_expr_right VALUES
+ (1, 1, '2024-01-01 08:00:00', 'r1'),
+ (2, 1, '2024-01-01 09:00:00', 'r2'),
+ (3, 1, '2024-01-01 10:00:00', 'r3'),
+ (4, 1, '2024-01-01 11:00:00', 'r4')
+ """
+
+ // Test 1: Expression on build side - r.ts + INTERVAL 1 HOUR
+ // l.ts >= r.ts + 1h => r.ts <= l.ts - 1h
+ // l.id=1 (10:00): r.ts <= 09:00 => match r2(09:00)
+ // l.id=2 (11:00): r.ts <= 10:00 => match r3(10:00)
+ // l.id=3 (12:00): r.ts <= 11:00 => match r4(11:00)
+ qt_expr_build_interval """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Test 2: Expression on probe side
+ qt_expr_probe_interval """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts - INTERVAL 1 HOUR >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Test 3: Simple column reference (baseline)
+ qt_expr_simple """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 4: DATE_ADD function ====================
+ // DATE_ADD is equivalent to INTERVAL but as a function
+ qt_expr_date_add """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= DATE_ADD(r.ts, INTERVAL 1 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 5: DATE_SUB function ====================
+ qt_expr_date_sub """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(DATE_SUB(l.ts, INTERVAL 1 HOUR) >= r.ts)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 6: DAYS_ADD function ====================
+ qt_expr_days_add """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= DAYS_ADD(r.ts, 0))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 7: Nested functions ====================
+ // HOURS_ADD inside MINUTES_ADD - tests nested function evaluation
+ qt_expr_nested_func """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= MINUTES_ADD(HOURS_ADD(r.ts, 1), -30))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 8: HOURS_ADD function ====================
+ qt_expr_hours_add """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= HOURS_ADD(r.ts, 2))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 9: Multiple intervals combined ====================
+ qt_expr_multi_interval """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 30 MINUTE + INTERVAL 30 MINUTE)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 10: Large dataset with expressions ====================
+ // Test with more rows to verify linear scan works correctly
+ sql "DROP TABLE IF EXISTS asof_expr_large_left"
+ sql "DROP TABLE IF EXISTS asof_expr_large_right"
+
+ sql """
+ CREATE TABLE asof_expr_large_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_expr_large_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // Insert probe rows
+ sql """
+ INSERT INTO asof_expr_large_left VALUES
+ (1, 1, '2024-01-01 12:00:00'),
+ (2, 1, '2024-01-01 15:00:00'),
+ (3, 1, '2024-01-01 18:00:00')
+ """
+
+ // Insert 15 build rows to exceed binary search threshold (>8)
+ sql """
+ INSERT INTO asof_expr_large_right VALUES
+ (1, 1, '2024-01-01 06:00:00', 'r01'),
+ (2, 1, '2024-01-01 07:00:00', 'r02'),
+ (3, 1, '2024-01-01 08:00:00', 'r03'),
+ (4, 1, '2024-01-01 09:00:00', 'r04'),
+ (5, 1, '2024-01-01 10:00:00', 'r05'),
+ (6, 1, '2024-01-01 11:00:00', 'r06'),
+ (7, 1, '2024-01-01 12:00:00', 'r07'),
+ (8, 1, '2024-01-01 13:00:00', 'r08'),
+ (9, 1, '2024-01-01 14:00:00', 'r09'),
+ (10, 1, '2024-01-01 15:00:00', 'r10'),
+ (11, 1, '2024-01-01 16:00:00', 'r11'),
+ (12, 1, '2024-01-01 17:00:00', 'r12'),
+ (13, 1, '2024-01-01 18:00:00', 'r13'),
+ (14, 1, '2024-01-01 19:00:00', 'r14'),
+ (15, 1, '2024-01-01 20:00:00', 'r15')
+ """
+
+ // With expression: l.ts >= r.ts + 3h => r.ts <= l.ts - 3h
+ // l.id=1 (12:00): r.ts <= 09:00 => match r04(09:00)
+ // l.id=2 (15:00): r.ts <= 12:00 => match r07(12:00)
+ // l.id=3 (18:00): r.ts <= 15:00 => match r10(15:00)
+ qt_expr_large_dataset """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_large_left l
+ ASOF LEFT JOIN asof_expr_large_right r
+ MATCH_CONDITION(l.ts >= DATE_ADD(r.ts, INTERVAL 3 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 11: Strict inequality with function ====================
+ // l.ts > r.ts + 3h => r.ts < l.ts - 3h
+ // l.id=1 (12:00): r.ts < 09:00 => match r03(08:00)
+ // l.id=2 (15:00): r.ts < 12:00 => match r06(11:00)
+ // l.id=3 (18:00): r.ts < 15:00 => match r09(14:00)
+ qt_expr_large_strict """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_large_left l
+ ASOF LEFT JOIN asof_expr_large_right r
+ MATCH_CONDITION(l.ts > DATE_ADD(r.ts, INTERVAL 3 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 12: <= with function ====================
+ // l.ts <= r.ts - 3h => r.ts >= l.ts + 3h (find smallest)
+ // l.id=1 (12:00): r.ts >= 15:00 => match r10(15:00)
+ // l.id=2 (15:00): r.ts >= 18:00 => match r13(18:00)
+ // l.id=3 (18:00): r.ts >= 21:00 => no match
+ qt_expr_large_le """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_large_left l
+ ASOF LEFT JOIN asof_expr_large_right r
+ MATCH_CONDITION(l.ts <= DATE_SUB(r.ts, INTERVAL 3 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 13: < with function ====================
+ // l.ts < r.ts - 3h => r.ts > l.ts + 3h (find smallest, strict)
+ // l.id=1 (12:00): r.ts > 15:00 => match r11(16:00)
+ // l.id=2 (15:00): r.ts > 18:00 => match r14(19:00)
+ // l.id=3 (18:00): r.ts > 21:00 => no match
+ qt_expr_large_lt """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_large_left l
+ ASOF LEFT JOIN asof_expr_large_right r
+ MATCH_CONDITION(l.ts < DATE_SUB(r.ts, INTERVAL 3 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 14: Expression with no matches ====================
+ // Very large offset - no matches expected
+ qt_expr_no_match """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts
+ FROM asof_expr_large_left l
+ ASOF LEFT JOIN asof_expr_large_right r
+ MATCH_CONDITION(l.ts >= DATE_ADD(r.ts, INTERVAL 24 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 15: INNER JOIN with expression ====================
+ qt_expr_inner """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_large_left l
+ ASOF INNER JOIN asof_expr_large_right r
+ MATCH_CONDITION(l.ts >= DATE_ADD(r.ts, INTERVAL 3 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 16: Multiple groups with expression ====================
+ sql "DROP TABLE IF EXISTS asof_expr_multigrp_left"
+ sql "DROP TABLE IF EXISTS asof_expr_multigrp_right"
+
+ sql """
+ CREATE TABLE asof_expr_multigrp_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_expr_multigrp_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_expr_multigrp_left VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 2, '2024-01-01 10:00:00'),
+ (3, 3, '2024-01-01 10:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_expr_multigrp_right VALUES
+ (1, 1, '2024-01-01 08:00:00', 'g1_r1'),
+ (2, 1, '2024-01-01 09:00:00', 'g1_r2'),
+ (3, 2, '2024-01-01 07:00:00', 'g2_r1'),
+ (4, 2, '2024-01-01 08:30:00', 'g2_r2'),
+ (5, 3, '2024-01-01 09:30:00', 'g3_r1')
+ """
+
+ // l.ts >= r.ts + 1h => r.ts <= l.ts - 1h = 09:00
+ // grp=1: r.ts <= 09:00 => match r2(09:00)
+ // grp=2: r.ts <= 09:00 => match r4(08:30)
+ // grp=3: r.ts <= 09:00 => no match (r5 is 09:30)
+ qt_expr_multigrp """
+ SELECT l.id, l.grp, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_multigrp_left l
+ ASOF LEFT JOIN asof_expr_multigrp_right r
+ MATCH_CONDITION(l.ts >= DATE_ADD(r.ts, INTERVAL 1 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 17: Expression on both sides ====================
+ qt_expr_both_sides """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(DATE_ADD(l.ts, INTERVAL 30 MINUTE) >= DATE_ADD(r.ts, INTERVAL 1 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 18: Deeply nested functions ====================
+ // SECONDS_ADD(MINUTES_ADD(HOURS_ADD(...))) - 3 levels of nesting
+ qt_expr_deep_nested """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= SECONDS_ADD(MINUTES_ADD(HOURS_ADD(r.ts, 1), -30), 0))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 19: MINUTES_ADD function ====================
+ qt_expr_minutes_add """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_left l
+ ASOF LEFT JOIN asof_expr_right r
+ MATCH_CONDITION(l.ts >= MINUTES_ADD(r.ts, 90))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 20: Edge case - expression result equals probe value ====================
+ sql "DROP TABLE IF EXISTS asof_expr_edge_left"
+ sql "DROP TABLE IF EXISTS asof_expr_edge_right"
+
+ sql """
+ CREATE TABLE asof_expr_edge_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_expr_edge_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_expr_edge_left VALUES
+ (1, 1, '2024-01-01 10:00:00')
+ """
+
+ // r.ts + 1h = exactly 10:00 for r1
+ sql """
+ INSERT INTO asof_expr_edge_right VALUES
+ (1, 1, '2024-01-01 09:00:00', 'exact_match'),
+ (2, 1, '2024-01-01 08:00:00', 'before'),
+ (3, 1, '2024-01-01 10:00:00', 'after')
+ """
+
+ // l.ts >= r.ts + 1h
+ // r1: 09:00 + 1h = 10:00, 10:00 >= 10:00 TRUE (exact match)
+ // r2: 08:00 + 1h = 09:00, 10:00 >= 09:00 TRUE
+ // r3: 10:00 + 1h = 11:00, 10:00 >= 11:00 FALSE
+ // Best match (largest r.ts satisfying): r1(09:00)
+ qt_expr_edge_exact """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_edge_left l
+ ASOF LEFT JOIN asof_expr_edge_right r
+ MATCH_CONDITION(l.ts >= r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // Strict: l.ts > r.ts + 1h
+ // r1: 10:00 > 10:00 FALSE
+ // r2: 10:00 > 09:00 TRUE
+ // Best: r2(08:00)
+ qt_expr_edge_strict """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_edge_left l
+ ASOF LEFT JOIN asof_expr_edge_right r
+ MATCH_CONDITION(l.ts > r.ts + INTERVAL 1 HOUR)
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+ // ==================== Test 21: Expression with NULL values ====================
+ sql "DROP TABLE IF EXISTS asof_expr_null_left"
+ sql "DROP TABLE IF EXISTS asof_expr_null_right"
+
+ sql """
+ CREATE TABLE asof_expr_null_left (
+ id INT,
+ grp INT,
+ ts DATETIME
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_expr_null_right (
+ id INT,
+ grp INT,
+ ts DATETIME,
+ data VARCHAR(20)
+ ) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ INSERT INTO asof_expr_null_left VALUES
+ (1, 1, '2024-01-01 10:00:00'),
+ (2, 1, NULL),
+ (3, 1, '2024-01-01 12:00:00')
+ """
+
+ sql """
+ INSERT INTO asof_expr_null_right VALUES
+ (1, 1, '2024-01-01 08:00:00', 'r1'),
+ (2, 1, NULL, 'r_null'),
+ (3, 1, '2024-01-01 09:00:00', 'r3')
+ """
+
+ // NULL in expression should not match
+ qt_expr_null_probe """
+ SELECT l.id, l.ts, r.id as rid, r.ts as rts, r.data
+ FROM asof_expr_null_left l
+ ASOF LEFT JOIN asof_expr_null_right r
+ MATCH_CONDITION(l.ts >= DATE_ADD(r.ts, INTERVAL 1 HOUR))
+ ON l.grp = r.grp
+ ORDER BY l.id
+ """
+
+}
diff --git a/regression-test/suites/nereids_p0/join/asof/test_asof_join_large_bucket.groovy b/regression-test/suites/nereids_p0/join/asof/test_asof_join_large_bucket.groovy
new file mode 100644
index 00000000000..fffd3c36f91
--- /dev/null
+++ b/regression-test/suites/nereids_p0/join/asof/test_asof_join_large_bucket.groovy
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Regression test for ASOF JOIN with many entries per hash bucket.
+// Covers two bugs:
+// 1. AsofEntry containing vectorized::Field crashed during pdqsort swap
+// (type not supported, type=INVALID)
+// 2. Concurrent probe tasks sorting same bucket entries without mutex (SIGSEGV)
+// Uses 50 probe rows and 30 build rows per group with nullable ASOF columns
+// to exercise pdqsort sorting path and NULL filtering.
+
+suite("test_asof_join_large_bucket", "nereids_p0") {
+ sql """ DROP TABLE IF EXISTS asof_lb_left """
+ sql """ DROP TABLE IF EXISTS asof_lb_right """
+
+ sql """
+ CREATE TABLE asof_lb_left (
+ id INT,
+ grp INT,
+ ts DATETIME NULL
+ ) DISTRIBUTED BY HASH(id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ sql """
+ CREATE TABLE asof_lb_right (
+ id INT,
+ grp INT,
+ ts DATETIME NULL,
+ val INT
+ ) DISTRIBUTED BY HASH(id) BUCKETS 3
+ PROPERTIES("replication_num" = "1")
+ """
+
+ // 50 probe rows per group (3 groups) + 1 NULL row per group = 153 rows
+ def left_vals = []
+ def lid = 1
+ for (g in 1..3) {
+ for (i in 0..49) {
+ left_vals.add("(${lid++}, ${g}, '2024-01-01 10:${String.format('%02d', i)}:00')")
+ }
+ }
+ left_vals.add("(${lid++}, 1, NULL)")
+ left_vals.add("(${lid++}, 2, NULL)")
+ left_vals.add("(${lid++}, 3, NULL)")
+ sql "INSERT INTO asof_lb_left VALUES " + left_vals.join(", ")
+
+ // 30 build rows per group (every 2 minutes, 0..58) + 1 NULL row per group = 93 rows
+ def right_vals = []
+ def rid = 1
+ for (g in 1..3) {
+ for (i in 0..29) {
+ def minute = i * 2
+ right_vals.add("(${rid}, ${g}, '2024-01-01 10:${String.format('%02d', minute)}:00', ${g * 1000 + rid})")
+ rid++
+ }
+ }
+ right_vals.add("(${rid++}, 1, NULL, 9001)")
+ right_vals.add("(${rid++}, 2, NULL, 9002)")
+ right_vals.add("(${rid++}, 3, NULL, 9003)")
+ sql "INSERT INTO asof_lb_right VALUES " + right_vals.join(", ")
+
+ // ==================== Test 1: ASOF LEFT JOIN COUNT(*) ====================
+ // Mimics the original failing query: SELECT COUNT(*) FROM t1 ASOF LEFT JOIN t2 ...
+ // NULL probe rows get NULL on right side (included in LEFT JOIN count)
+ qt_lb_left_count """
+ SELECT l.grp, COUNT(*) as cnt
+ FROM asof_lb_left l
+ ASOF LEFT JOIN asof_lb_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ GROUP BY l.grp
+ ORDER BY l.grp
+ """
+
+ // ==================== Test 2: ASOF INNER JOIN COUNT(*) ====================
+ // NULL probe rows excluded from INNER JOIN
+ qt_lb_inner_count """
+ SELECT l.grp, COUNT(*) as cnt
+ FROM asof_lb_left l
+ ASOF INNER JOIN asof_lb_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ GROUP BY l.grp
+ ORDER BY l.grp
+ """
+
+ // ==================== Test 3: Spot-check boundary and NULL rows ====================
+ qt_lb_spot_check """
+ SELECT l.id, l.grp, l.ts, r.id as rid, r.ts as rts, r.val
+ FROM asof_lb_left l
+ ASOF LEFT JOIN asof_lb_right r
+ MATCH_CONDITION(l.ts >= r.ts)
+ ON l.grp = r.grp
+ WHERE l.id IN (1, 25, 50, 51, 100, 151, 152)
+ ORDER BY l.id
+ """
+
+ // ==================== Test 4: <= operator with many entries ====================
+ qt_lb_le_count """
+ SELECT l.grp, COUNT(*) as cnt
+ FROM asof_lb_left l
+ ASOF LEFT JOIN asof_lb_right r
+ MATCH_CONDITION(l.ts <= r.ts)
+ ON l.grp = r.grp
+ GROUP BY l.grp
+ ORDER BY l.grp
+ """
+
+ // ==================== Test 5: Strict > operator ====================
+ qt_lb_gt_count """
+ SELECT l.grp, COUNT(*) as cnt
+ FROM asof_lb_left l
+ ASOF LEFT JOIN asof_lb_right r
+ MATCH_CONDITION(l.ts > r.ts)
+ ON l.grp = r.grp
+ GROUP BY l.grp
+ ORDER BY l.grp
+ """
+
+ // ==================== Test 6: Strict < operator ====================
+ qt_lb_lt_count """
+ SELECT l.grp, COUNT(*) as cnt
+ FROM asof_lb_left l
+ ASOF LEFT JOIN asof_lb_right r
+ MATCH_CONDITION(l.ts < r.ts)
+ ON l.grp = r.grp
+ GROUP BY l.grp
+ ORDER BY l.grp
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org