You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2025/11/06 03:25:14 UTC
(doris) 01/18: support prune nested columns
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6ccc8ca260ad1ac6a1ba67d157a41923aa2cde69
Author: 924060929 <la...@selectdb.com>
AuthorDate: Wed Sep 24 17:04:44 2025 +0800
support prune nested columns
---
.idea/vcs.xml | 52 +-
be/src/olap/iterators.h | 5 +
be/src/olap/rowset/beta_rowset_reader.cpp | 4 +
be/src/olap/rowset/rowset_reader_context.h | 5 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 361 ++++++++++-
be/src/olap/rowset/segment_v2/column_reader.h | 70 ++-
be/src/olap/rowset/segment_v2/segment.cpp | 16 +
be/src/olap/tablet_reader.cpp | 3 +
be/src/olap/tablet_reader.h | 4 +
be/src/olap/tablet_schema.cpp | 16 +
be/src/olap/tablet_schema.h | 10 +
be/src/runtime/descriptors.cpp | 5 +
be/src/runtime/descriptors.h | 8 +
be/src/vec/columns/column_map.cpp | 1 +
be/src/vec/exec/scan/olap_scanner.cpp | 20 +-
.../olap/rowset/segment_v2/column_reader_test.cpp | 220 +++++++
.../org/apache/doris/analysis/AccessPathInfo.java | 38 ++
.../org/apache/doris/analysis/SlotDescriptor.java | 43 ++
.../org/apache/doris/datasource/FileScanNode.java | 10 +-
.../org/apache/doris/nereids/StatementContext.java | 10 +
.../glue/translator/PhysicalPlanTranslator.java | 5 +-
.../glue/translator/PlanTranslatorContext.java | 18 +
.../doris/nereids/jobs/executor/Rewriter.java | 6 +
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 7 +
.../doris/nereids/rules/analysis/BindRelation.java | 9 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 10 +
.../rules/expression/ExpressionNormalization.java | 2 +
.../rules/expression/ExpressionRuleType.java | 1 +
.../expression/rules/NormalizeStructElement.java | 66 ++
.../rewrite/AccessPathExpressionCollector.java | 511 ++++++++++++++++
.../rules/rewrite/AccessPathPlanCollector.java | 176 ++++++
.../nereids/rules/rewrite/NestedColumnPruning.java | 487 +++++++++++++++
.../rules/rewrite/PushDownNestedColumn.java | 315 ++++++++++
.../nereids/rules/rewrite/SlotTypeReplacer.java | 674 +++++++++++++++++++++
.../rules/rewrite/VariantSubPathPruning.java | 3 +-
.../trees/expressions/PreferPushDownProject.java | 41 ++
.../nereids/trees/expressions/SlotReference.java | 56 +-
.../expressions/functions/scalar/ArrayFirst.java | 6 +
.../expressions/functions/scalar/ArrayLast.java | 6 +
.../expressions/functions/scalar/ArrayMap.java | 3 +-
.../expressions/functions/scalar/ElementAt.java | 4 +-
.../functions/scalar/HighOrderFunction.java | 3 +-
.../functions/scalar/MapContainsEntry.java | 3 +-
.../functions/scalar/MapContainsKey.java | 3 +-
.../functions/scalar/MapContainsValue.java | 3 +-
.../expressions/functions/scalar/MapEntries.java | 3 +-
.../expressions/functions/scalar/MapKeys.java | 3 +-
.../expressions/functions/scalar/MapValues.java | 3 +-
.../functions/scalar/StructElement.java | 9 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 14 +-
.../trees/plans/logical/LogicalFileScan.java | 44 +-
.../trees/plans/logical/LogicalHudiScan.java | 22 +-
.../trees/plans/logical/LogicalOlapScan.java | 17 +
.../trees/plans/logical/LogicalTVFRelation.java | 23 +-
.../org/apache/doris/nereids/types/ArrayType.java | 2 +-
.../org/apache/doris/nereids/types/MapType.java | 2 +-
.../doris/nereids/types/NestedColumnPrunable.java | 22 +
.../apache/doris/nereids/types/StructField.java | 3 +-
.../org/apache/doris/nereids/types/StructType.java | 10 +-
.../org/apache/doris/planner/OlapScanNode.java | 3 +
.../java/org/apache/doris/planner/PlanNode.java | 4 +-
.../java/org/apache/doris/planner/ScanNode.java | 109 ++++
.../java/org/apache/doris/qe/SessionVariable.java | 9 +
.../rules/rewrite/PruneNestedColumnTest.java | 625 +++++++++++++++++++
.../apache/doris/utframe/TestWithFeService.java | 7 +-
gensrc/thrift/Descriptors.thrift | 48 +-
.../complex_types/test_pruned_columns.out | 24 +
.../complex_types/test_pruned_columns.groovy | 61 ++
69 files changed, 4285 insertions(+), 103 deletions(-)
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 0a185d9bdc7..8b083db2f94 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -1,34 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
<project version="4">
- <component name="IssueNavigationConfiguration">
- <option name="links">
- <list>
- <IssueNavigationLink>
- <option name="issueRegexp" value="#(\d+)" />
- <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
- </IssueNavigationLink>
- </list>
- </option>
- </component>
- <component name="VcsDirectoryMappings">
- <mapping directory="$PROJECT_DIR$" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
- </component>
-</project>
+ <component name="IssueNavigationConfiguration">
+ <option name="links">
+ <list>
+ <IssueNavigationLink>
+ <option name="issueRegexp" value="#(\d+)" />
+ <option name="linkRegexp" value="https://github.com/apache/doris/pull/$1" />
+ </IssueNavigationLink>
+ </list>
+ </option>
+ </component>
+ <component name="VcsDirectoryMappings">
+ <mapping directory="$PROJECT_DIR$" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/faiss" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/openblas" vcs="Git" />
+ </component>
+</project>
\ No newline at end of file
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index 9a3ed4ea998..1b63aad4909 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -47,6 +47,8 @@ namespace segment_v2 {
struct SubstreamIterator;
}
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class StorageReadOptions {
public:
struct KeyRange {
@@ -143,6 +145,9 @@ public:
std::map<ColumnId, size_t> vir_cid_to_idx_in_block;
std::map<size_t, vectorized::DataTypePtr> vir_col_idx_to_type;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::shared_ptr<vectorized::ScoreRuntime> score_runtime;
CollectionStatisticsPtr collection_statistics;
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 1de17e27eee..28683540365 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -103,6 +103,10 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
_read_options.remaining_conjunct_roots = _read_context->remaining_conjunct_roots;
_read_options.common_expr_ctxs_push_down = _read_context->common_expr_ctxs_push_down;
_read_options.virtual_column_exprs = _read_context->virtual_column_exprs;
+
+ _read_options.all_access_paths = _read_context->all_access_paths;
+ _read_options.predicate_access_paths = _read_context->predicate_access_paths;
+
_read_options.ann_topn_runtime = _read_context->ann_topn_runtime;
_read_options.vir_cid_to_idx_in_block = _read_context->vir_cid_to_idx_in_block;
_read_options.vir_col_idx_to_type = _read_context->vir_col_idx_to_type;
diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h
index 951679f413b..fffa16d0f5c 100644
--- a/be/src/olap/rowset/rowset_reader_context.h
+++ b/be/src/olap/rowset/rowset_reader_context.h
@@ -18,6 +18,8 @@
#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
+#include <vector>
+
#include "io/io_common.h"
#include "olap/column_predicate.h"
#include "olap/olap_common.h"
@@ -90,6 +92,9 @@ struct RowsetReaderContext {
std::map<ColumnId, size_t> vir_cid_to_idx_in_block;
std::map<size_t, vectorized::DataTypePtr> vir_col_idx_to_type;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::shared_ptr<vectorized::ScoreRuntime> score_runtime;
CollectionStatisticsPtr collection_statistics;
std::shared_ptr<segment_v2::AnnTopNRuntime> ann_topn_runtime;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 16d87a1d425..9c68eee2fea 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -18,6 +18,7 @@
#include "olap/rowset/segment_v2/column_reader.h"
#include <assert.h>
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/segment_v2.pb.h>
#include <algorithm>
@@ -72,11 +73,11 @@
#include "vec/columns/column_map.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_struct.h"
-#include "vec/columns/column_variant.h"
#include "vec/columns/column_vector.h"
#include "vec/common/assert_cast.h"
#include "vec/common/schema_util.h"
#include "vec/common/string_ref.h"
+#include "vec/common/typeid_cast.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type_agg_state.h"
#include "vec/data_types/data_type_factory.hpp"
@@ -804,11 +805,12 @@ Status ColumnReader::new_iterator(ColumnIteratorUPtr* iterator, const TabletColu
*iterator = std::make_unique<EmptyFileColumnIterator>();
return Status::OK();
}
- if (is_scalar_type((FieldType)_meta_type)) {
+ if (is_scalar_type(_meta_type)) {
*iterator = std::make_unique<FileColumnIterator>(shared_from_this());
+ (*iterator)->set_column_name(tablet_column ? tablet_column->name() : "");
return Status::OK();
} else {
- auto type = (FieldType)_meta_type;
+ auto type = _meta_type;
switch (type) {
case FieldType::OLAP_FIELD_TYPE_AGG_STATE: {
return new_agg_state_iterator(iterator);
@@ -842,6 +844,8 @@ Status ColumnReader::new_array_iterator(ColumnIteratorUPtr* iterator,
? &tablet_column->get_sub_column(0)
: nullptr));
+ item_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : "");
+
ColumnIteratorUPtr offset_iterator;
RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&offset_iterator, nullptr));
auto* file_iter = static_cast<FileColumnIterator*>(offset_iterator.release());
@@ -865,11 +869,13 @@ Status ColumnReader::new_map_iterator(ColumnIteratorUPtr* iterator,
&key_iterator, tablet_column && tablet_column->get_subtype_count() > 1
? &tablet_column->get_sub_column(0)
: nullptr));
+ key_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(0).name() : "");
ColumnIteratorUPtr val_iterator;
RETURN_IF_ERROR(_sub_readers[1]->new_iterator(
&val_iterator, tablet_column && tablet_column->get_subtype_count() > 1
? &tablet_column->get_sub_column(1)
: nullptr));
+ val_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(1).name() : "");
ColumnIteratorUPtr offsets_iterator;
RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&offsets_iterator, nullptr));
auto* file_iter = static_cast<FileColumnIterator*>(offsets_iterator.release());
@@ -897,6 +903,8 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
ColumnIteratorUPtr sub_column_iterator;
RETURN_IF_ERROR(_sub_readers[i]->new_iterator(
&sub_column_iterator, tablet_column ? &tablet_column->get_sub_column(i) : nullptr));
+ sub_column_iterator->set_column_name(tablet_column ? tablet_column->get_sub_column(i).name()
+ : "");
sub_column_iterators.emplace_back(std::move(sub_column_iterator));
}
@@ -917,6 +925,33 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
return Status::OK();
}
+Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
+ const TColumnAccessPaths& access_paths) {
+ TColumnAccessPaths sub_access_paths = access_paths;
+ for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) {
+ TColumnAccessPath& name_path = *it;
+ if (name_path.data_access_path.path.empty()) {
+ return ResultError(
+ Status::InternalError("Invalid access path for struct column: path is empty"));
+ }
+
+ if (name_path.data_access_path.path[0] != _column_name) {
+ return ResultError(Status::InternalError(
+ R"(Invalid access path for column: expected name "{}", got "{}")", _column_name,
+ name_path.data_access_path.path[0]));
+ }
+
+ name_path.data_access_path.path.erase(name_path.data_access_path.path.begin());
+ if (!name_path.data_access_path.path.empty()) {
+ ++it;
+ } else {
+ set_need_to_read();
+ it = sub_access_paths.erase(it);
+ }
+ }
+ return sub_access_paths;
+}
+
///====================== MapFileColumnIterator ============================////
MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr<ColumnReader> reader,
ColumnIteratorUPtr null_iterator,
@@ -933,6 +968,10 @@ MapFileColumnIterator::MapFileColumnIterator(std::shared_ptr<ColumnReader> reade
}
Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
RETURN_IF_ERROR(_key_iterator->init(opts));
RETURN_IF_ERROR(_val_iterator->init(opts));
RETURN_IF_ERROR(_offsets_iterator->init(opts));
@@ -943,6 +982,11 @@ Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
}
Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
if (_map_reader->is_nullable()) {
RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
}
@@ -957,10 +1001,16 @@ Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
- const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Map column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
+ auto& column_map = assert_cast<vectorized::ColumnMap&>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
- auto column_offsets_ptr = column_map->get_offsets_column().assume_mutable();
+ auto column_offsets_ptr = column_map.get_offsets_column().assume_mutable();
bool offsets_has_null = false;
ssize_t start = column_offsets_ptr->size();
RETURN_IF_ERROR(_offsets_iterator->next_batch(n, column_offsets_ptr, &offsets_has_null));
@@ -973,8 +1023,8 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
DCHECK(column_offsets.get_data().back() >= column_offsets.get_data()[start - 1]);
size_t num_items =
column_offsets.get_data().back() - column_offsets.get_data()[start - 1]; // -1 is valid
- auto key_ptr = column_map->get_keys().assume_mutable();
- auto val_ptr = column_map->get_values().assume_mutable();
+ auto key_ptr = column_map.get_keys().assume_mutable();
+ auto val_ptr = column_map.get_values().assume_mutable();
if (num_items > 0) {
size_t num_read = num_items;
@@ -983,6 +1033,9 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, key_ptr, &key_has_null));
RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, val_ptr, &val_has_null));
DCHECK(num_read == num_items);
+
+ column_map.get_keys_ptr() = std::move(key_ptr);
+ column_map.get_values_ptr() = std::move(val_ptr);
}
if (dst->is_nullable()) {
@@ -1008,6 +1061,12 @@ Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr
Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
size_t num_read = 1;
@@ -1017,6 +1076,93 @@ Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t
return Status::OK();
}
+void MapFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ _key_iterator->set_need_to_read();
+ _val_iterator->set_need_to_read();
+}
+
+void MapFileColumnIterator::remove_pruned_sub_iterators() {
+ _key_iterator->remove_pruned_sub_iterators();
+ _val_iterator->remove_pruned_sub_iterators();
+}
+
+Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Map column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ if (sub_all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ TColumnAccessPaths key_all_access_paths;
+ TColumnAccessPaths val_all_access_paths;
+ TColumnAccessPaths key_predicate_access_paths;
+ TColumnAccessPaths val_predicate_access_paths;
+
+ for (auto paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
+ }
+ }
+ const auto need_read_keys = !key_all_access_paths.empty();
+ const auto need_read_values = !val_all_access_paths.empty();
+
+ for (auto paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
+ }
+ }
+
+ if (need_read_keys) {
+ _key_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _key_iterator->set_access_paths(key_all_access_paths, key_predicate_access_paths));
+ } else {
+ _key_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Map column iterator set key column to SKIP_READING";
+ }
+
+ if (need_read_values) {
+ _val_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _val_iterator->set_access_paths(val_all_access_paths, val_predicate_access_paths));
+ } else {
+ _val_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Map column iterator set value column to SKIP_READING";
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
StructFileColumnIterator::StructFileColumnIterator(
@@ -1029,6 +1175,11 @@ StructFileColumnIterator::StructFileColumnIterator(
}
Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
for (auto& column_iterator : _sub_column_iterators) {
RETURN_IF_ERROR(column_iterator->init(opts));
}
@@ -1040,16 +1191,23 @@ Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
- const auto* column_struct = vectorized::check_and_get_column<vectorized::ColumnStruct>(
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
+ auto& column_struct = assert_cast<vectorized::ColumnStruct&>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
- for (size_t i = 0; i < column_struct->tuple_size(); i++) {
+ for (size_t i = 0; i < column_struct.tuple_size(); i++) {
size_t num_read = *n;
- auto sub_column_ptr = column_struct->get_column(i).assume_mutable();
+ auto sub_column_ptr = column_struct.get_column(i).assume_mutable();
bool column_has_null = false;
RETURN_IF_ERROR(
_sub_column_iterators[i]->next_batch(&num_read, sub_column_ptr, &column_has_null));
DCHECK(num_read == *n);
+ column_struct.get_column_ptr(i) = std::move(sub_column_ptr);
}
if (dst->is_nullable()) {
@@ -1075,6 +1233,11 @@ Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumn
}
Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
for (auto& column_iterator : _sub_column_iterators) {
RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord));
}
@@ -1086,6 +1249,12 @@ Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
size_t num_read = 1;
@@ -1095,6 +1264,83 @@ Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const siz
return Status::OK();
}
+void StructFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ for (auto& sub_iterator : _sub_column_iterators) {
+ sub_iterator->set_need_to_read();
+ }
+}
+
+void StructFileColumnIterator::remove_pruned_sub_iterators() {
+ for (auto it = _sub_column_iterators.begin(); it != _sub_column_iterators.end();) {
+ auto& sub_iterator = *it;
+ if (sub_iterator->reading_flag() == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Struct column iterator remove pruned sub-column "
+ << sub_iterator->column_name();
+ it = _sub_column_iterators.erase(it);
+ } else {
+ sub_iterator->remove_pruned_sub_iterators();
+ ++it;
+ }
+ }
+}
+
+Status StructFileColumnIterator::set_access_paths(
+ const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Struct column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
+
+ for (auto& sub_iterator : _sub_column_iterators) {
+ const auto name = sub_iterator->column_name();
+ bool need_to_read = no_sub_column_to_skip;
+ TColumnAccessPaths sub_all_access_paths_of_this;
+ if (!need_to_read) {
+ for (const auto& paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_all_access_paths_of_this.emplace_back(paths);
+ }
+ }
+ need_to_read = !sub_all_access_paths_of_this.empty();
+ }
+
+ if (!need_to_read) {
+ set_reading_flag(ReadingFlag::SKIP_READING);
+ sub_iterator->set_reading_flag(ReadingFlag::SKIP_READING);
+ LOG(INFO) << "Struct column iterator set sub-column " << name << " to SKIP_READING";
+ continue;
+ }
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ sub_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+
+ TColumnAccessPaths sub_predicate_access_paths_of_this;
+
+ if (!no_predicate_sub_column) {
+ for (const auto& paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_predicate_access_paths_of_this.emplace_back(paths);
+ }
+ }
+ }
+
+ RETURN_IF_ERROR(sub_iterator->set_access_paths(sub_all_access_paths_of_this,
+ sub_predicate_access_paths_of_this));
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
Status OffsetFileColumnIterator::init(const ColumnIteratorOptions& opts) {
RETURN_IF_ERROR(_offset_iterator->init(opts));
@@ -1166,6 +1412,11 @@ ArrayFileColumnIterator::ArrayFileColumnIterator(std::shared_ptr<ColumnReader> r
}
Status ArrayFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "Array column iterator column " << _column_name << " skip readking.";
+ return Status::OK();
+ }
+
RETURN_IF_ERROR(_offset_iterator->init(opts));
RETURN_IF_ERROR(_item_iterator->init(opts));
if (_array_reader->is_nullable()) {
@@ -1183,6 +1434,11 @@ Status ArrayFileColumnIterator::_seek_by_offsets(ordinal_t ord) {
}
Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
RETURN_IF_ERROR(_offset_iterator->seek_to_ordinal(ord));
if (_array_reader->is_nullable()) {
RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
@@ -1192,6 +1448,12 @@ Status ArrayFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
const auto* column_array = vectorized::check_and_get_column<vectorized::ColumnArray>(
dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
: *dst);
@@ -1240,6 +1502,12 @@ Status ArrayFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnP
Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "Array column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
for (size_t i = 0; i < count; ++i) {
// TODO(cambyzju): now read array one by one, need optimize later
RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
@@ -1250,11 +1518,67 @@ Status ArrayFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size
return Status::OK();
}
+void ArrayFileColumnIterator::set_need_to_read() {
+ set_reading_flag(ReadingFlag::NEED_TO_READ);
+ _item_iterator->set_need_to_read();
+}
+
+void ArrayFileColumnIterator::remove_pruned_sub_iterators() {
+ _item_iterator->remove_pruned_sub_iterators();
+}
+
+Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (all_access_paths.empty()) {
+ return Status::OK();
+ }
+
+ if (!predicate_access_paths.empty()) {
+ set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
+ LOG(INFO) << "Array column iterator set sub-column " << _column_name
+ << " to READING_FOR_PREDICATE";
+ }
+
+ auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
+ auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
+
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
+
+ if (!no_sub_column_to_skip) {
+ for (auto& path : sub_all_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
+ }
+ }
+ }
+
+ if (!no_predicate_sub_column) {
+ for (auto& path : sub_predicate_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
+ }
+ }
+ }
+
+ if (!no_sub_column_to_skip || !no_predicate_sub_column) {
+ _item_iterator->set_reading_flag(ReadingFlag::NEED_TO_READ);
+ RETURN_IF_ERROR(
+ _item_iterator->set_access_paths(sub_all_access_paths, sub_predicate_access_paths));
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
FileColumnIterator::FileColumnIterator(std::shared_ptr<ColumnReader> reader) : _reader(reader) {}
Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ LOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
_opts = opts;
if (!_opts.use_page_cache) {
_reader->disable_index_meta_cache();
@@ -1286,6 +1610,11 @@ Status FileColumnIterator::init(const ColumnIteratorOptions& opts) {
FileColumnIterator::~FileColumnIterator() = default;
Status FileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ return Status::OK();
+ }
+
// if current page contains this row, we don't need to seek
if (!_page || !_page.contains(ord) || !_page_iter.valid()) {
RETURN_IF_ERROR(_reader->seek_at_or_before(ord, &_page_iter, _opts));
@@ -1336,6 +1665,12 @@ Status FileColumnIterator::next_batch_of_zone_map(size_t* n, vectorized::Mutable
Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(*n);
+ return Status::OK();
+ }
+
size_t curr_size = dst->byte_size();
dst->reserve(*n);
size_t remaining = *n;
@@ -1392,6 +1727,12 @@ Status FileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& d
Status FileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
vectorized::MutableColumnPtr& dst) {
+ if (_reading_flag == ReadingFlag::SKIP_READING) {
+ DLOG(INFO) << "File column iterator column " << _column_name << " skip reading.";
+ dst->resize(count);
+ return Status::OK();
+ }
+
size_t remaining = count;
size_t total_read_count = 0;
size_t nrows_to_read = 0;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 2ffdb2ee699..8e96e2f3faa 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/segment_v2.pb.h>
#include <sys/types.h>
@@ -69,6 +70,8 @@ class FileReader;
struct Slice;
struct StringRef;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
namespace segment_v2 {
class EncodingInfo;
@@ -271,7 +274,6 @@ private:
Status _calculate_row_ranges(const std::vector<uint32_t>& page_indexes, RowRanges* row_ranges,
const ColumnIteratorOptions& iter_opts);
-private:
int64_t _meta_length;
FieldType _meta_type;
FieldType _meta_children_column_type;
@@ -366,8 +368,54 @@ public:
virtual bool is_all_dict_encoding() const { return false; }
+ virtual Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) {
+ if (!predicate_access_paths.empty()) {
+ _reading_flag = ReadingFlag::READING_FOR_PREDICATE;
+ }
+ return Status::OK();
+ }
+
+ void set_column_name(const std::string& column_name) { _column_name = column_name; }
+
+ const std::string& column_name() const { return _column_name; }
+
+ // Since there may be multiple paths with conflicts or overlaps,
+ // we need to define several reading flags:
+ //
+ // NORMAL_READING — Default value, indicating that the column should be read.
+ // SKIP_READING — The column should not be read.
+ // NEED_TO_READ — The column must be read.
+ // READING_FOR_PREDICATE — The column is required for predicate evaluation.
+ //
+ // For example, suppose there are two paths:
+ // - Path 1 specifies that column A needs to be read, so it is marked as NEED_TO_READ.
+ // - Path 2 specifies that the column should not be read, but since it is already marked as NEED_TO_READ,
+ // it should not be changed to SKIP_READING.
+ enum class ReadingFlag : int {
+ NORMAL_READING,
+ SKIP_READING,
+ NEED_TO_READ,
+ READING_FOR_PREDICATE
+ };
+ void set_reading_flag(ReadingFlag flag) {
+ if (static_cast<int>(flag) > static_cast<int>(_reading_flag)) {
+ _reading_flag = flag;
+ }
+ }
+
+ ReadingFlag reading_flag() const { return _reading_flag; }
+
+ virtual void set_need_to_read() { set_reading_flag(ReadingFlag::NEED_TO_READ); }
+
+ virtual void remove_pruned_sub_iterators() {};
+
protected:
+ Result<TColumnAccessPaths> _get_sub_access_paths(const TColumnAccessPaths& access_paths);
ColumnIteratorOptions _opts;
+
+ ReadingFlag _reading_flag {ReadingFlag::NORMAL_READING};
+ std::string _column_name;
};
// This iterator is used to read column data from file
@@ -504,6 +552,13 @@ public:
return _offsets_iterator->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _map_reader = nullptr;
ColumnIteratorUPtr _null_iterator;
@@ -533,6 +588,13 @@ public:
return _sub_column_iterators[0]->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _struct_reader = nullptr;
ColumnIteratorUPtr _null_iterator;
@@ -561,6 +623,12 @@ public:
return _offset_iterator->get_current_ordinal();
}
+ Status set_access_paths(const TColumnAccessPaths& all_access_paths,
+ const TColumnAccessPaths& predicate_access_paths) override;
+ void set_need_to_read() override;
+
+ void remove_pruned_sub_iterators() override;
+
private:
std::shared_ptr<ColumnReader> _array_reader = nullptr;
std::unique_ptr<OffsetFileColumnIterator> _offset_iterator;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index d95d049c7cf..211ba10bbbb 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -17,12 +17,14 @@
#include "olap/rowset/segment_v2/segment.h"
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <gen_cpp/olap_file.pb.h>
#include <gen_cpp/segment_v2.pb.h>
#include <cstring>
#include <memory>
+#include <sstream>
#include <utility>
#include "cloud/config.h"
@@ -765,6 +767,20 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
sparse_column_cache_ptr));
} else {
RETURN_IF_ERROR(reader->new_iterator(iter, &tablet_column, opt));
+ if (opt->all_access_paths.contains(unique_id) ||
+ opt->predicate_access_paths.contains(unique_id)) {
+ const auto& all_access_paths = opt->all_access_paths.contains(unique_id)
+ ? opt->all_access_paths.at(unique_id)
+ : TColumnAccessPaths {};
+ const auto& predicate_access_paths = opt->predicate_access_paths.contains(unique_id)
+ ? opt->predicate_access_paths.at(unique_id)
+ : TColumnAccessPaths {};
+
+ // set column name to apply access paths.
+ (*iter)->set_column_name(tablet_column.name());
+ RETURN_IF_ERROR((*iter)->set_access_paths(all_access_paths, predicate_access_paths));
+ (*iter)->remove_pruned_sub_iterators();
+ }
}
if (config::enable_column_type_check && !tablet_column.has_path_info() &&
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 3660e709e22..3dc120ec4c4 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -263,6 +263,9 @@ Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) {
_reader_context.ann_topn_runtime = read_params.ann_topn_runtime;
_reader_context.condition_cache_digest = read_params.condition_cache_digest;
+ _reader_context.all_access_paths = read_params.all_access_paths;
+ _reader_context.predicate_access_paths = read_params.predicate_access_paths;
+
return Status::OK();
}
diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h
index 8630c932052..593afa9dbe5 100644
--- a/be/src/olap/tablet_reader.h
+++ b/be/src/olap/tablet_reader.h
@@ -17,6 +17,7 @@
#pragma once
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <stddef.h>
@@ -146,6 +147,9 @@ public:
// slots that cast may be eliminated in storage layer
std::map<std::string, vectorized::DataTypePtr> target_cast_type_for_variants;
+ std::map<int32_t, TColumnAccessPaths> all_access_paths;
+ std::map<int32_t, TColumnAccessPaths> predicate_access_paths;
+
std::vector<RowSetSplits> rs_splits;
// For unique key table with merge-on-write
DeleteBitmapPtr delete_bitmap = nullptr;
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 079cf9278ff..cee5f53ae6b 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1684,6 +1684,11 @@ vectorized::Block TabletSchema::create_block(
tablet_columns_need_convert_null->find(cid) !=
tablet_columns_need_convert_null->end());
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(col, is_nullable);
+ if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col.unique_id())) {
+ data_type = _pruned_columns_data_type.at(col.unique_id());
+ }
+ }
if (_vir_col_idx_to_unique_id.contains(cid)) {
block.insert({vectorized::ColumnNothing::create(0), data_type, col.name()});
VLOG_DEBUG << fmt::format(
@@ -1703,7 +1708,13 @@ vectorized::Block TabletSchema::create_block(bool ignore_dropped_col) const {
if (ignore_dropped_col && is_dropped_column(*col)) {
continue;
}
+
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(*col);
+ if (col->type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col->unique_id())) {
+ data_type = _pruned_columns_data_type.at(col->unique_id());
+ }
+ }
block.insert({data_type->create_column(), data_type, col->name()});
}
return block;
@@ -1714,6 +1725,11 @@ vectorized::Block TabletSchema::create_block_by_cids(const std::vector<uint32_t>
for (const auto& cid : cids) {
const auto& col = *_cols[cid];
auto data_type = vectorized::DataTypeFactory::instance().create_data_type(col);
+ if (col.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (_pruned_columns_data_type.contains(col.unique_id())) {
+ data_type = _pruned_columns_data_type.at(col.unique_id());
+ }
+ }
block.insert({data_type->create_column(), data_type, col.name()});
}
return block;
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 57463316cdf..9382a68c3a1 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -42,6 +42,7 @@
#include "runtime/define_primitive_type.h"
#include "runtime/descriptors.h"
#include "runtime/memory/lru_cache_policy.h"
+#include "udf/udf.h"
#include "util/debug_points.h"
#include "util/string_parser.hpp"
#include "util/string_util.h"
@@ -681,6 +682,14 @@ public:
return 0;
}
+ void add_pruned_columns_data_type(int32_t col_unique_id, vectorized::DataTypePtr data_type) {
+ _pruned_columns_data_type[col_unique_id] = std::move(data_type);
+ }
+
+ void clear_pruned_columns_data_type() { _pruned_columns_data_type.clear(); }
+
+ bool has_pruned_columns() const { return !_pruned_columns_data_type.empty(); }
+
private:
friend bool operator==(const TabletSchema& a, const TabletSchema& b);
friend bool operator!=(const TabletSchema& a, const TabletSchema& b);
@@ -751,6 +760,7 @@ private:
bool _enable_variant_flatten_nested = false;
std::map<size_t, int32_t> _vir_col_idx_to_unique_id;
+ std::map<int32_t, vectorized::DataTypePtr> _pruned_columns_data_type;
// value: extracted path set and sparse path set
std::unordered_map<int32_t, PathsSetInfo> _path_set_info_map;
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index 7ccc69ed9ac..a17f0926752 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -63,6 +63,11 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc)
_is_materialized(tdesc.isMaterialized && tdesc.need_materialize),
_is_key(tdesc.is_key),
_column_paths(tdesc.column_paths),
+ _all_access_paths(tdesc.__isset.all_access_paths ? tdesc.all_access_paths
+ : TColumnAccessPaths {}),
+ _predicate_access_paths(tdesc.__isset.predicate_access_paths
+ ? tdesc.predicate_access_paths
+ : TColumnAccessPaths {}),
_is_auto_increment(tdesc.__isset.is_auto_increment ? tdesc.is_auto_increment : false),
_col_default_value(tdesc.__isset.col_default_value ? tdesc.col_default_value : "") {
if (tdesc.__isset.virtual_column_expr) {
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 0d86844b226..c66ccd80b1d 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -53,6 +53,8 @@ class ObjectPool;
class PTupleDescriptor;
class PSlotDescriptor;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class SlotDescriptor {
public:
MOCK_DEFINE(virtual ~SlotDescriptor() = default;)
@@ -82,6 +84,9 @@ public:
bool is_key() const { return _is_key; }
const std::vector<std::string>& column_paths() const { return _column_paths; };
+ const TColumnAccessPaths& all_access_paths() const { return _all_access_paths; }
+ const TColumnAccessPaths& predicate_access_paths() const { return _predicate_access_paths; }
+
bool is_auto_increment() const { return _is_auto_increment; }
bool is_skip_bitmap_col() const { return _col_name == SKIP_BITMAP_COL; }
@@ -128,6 +133,9 @@ private:
const bool _is_key;
const std::vector<std::string> _column_paths;
+ const TColumnAccessPaths _all_access_paths;
+ const TColumnAccessPaths _predicate_access_paths;
+
const bool _is_auto_increment;
const std::string _col_default_value;
diff --git a/be/src/vec/columns/column_map.cpp b/be/src/vec/columns/column_map.cpp
index 8317aa5bb5e..7cd6bfe9f4a 100644
--- a/be/src/vec/columns/column_map.cpp
+++ b/be/src/vec/columns/column_map.cpp
@@ -59,6 +59,7 @@ ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values, Mutable
/// This will also prevent possible overflow in offset.
if (keys_column->size() != last_offset) {
+ DCHECK(0);
throw doris::Exception(
doris::ErrorCode::INTERNAL_ERROR,
"offsets_column size {} has data inconsistent with key_column {}", last_offset,
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index b1b67e8300d..51fc4daa669 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -83,6 +83,8 @@ OlapScanner::OlapScanner(pipeline::ScanLocalStateBase* parent, OlapScanner::Para
.function_filters {},
.delete_predicates {},
.target_cast_type_for_variants {},
+ .all_access_paths {},
+ .predicate_access_paths {},
.rs_splits {},
.return_columns {},
.output_columns {},
@@ -265,7 +267,7 @@ Status OlapScanner::prepare() {
// Add newly created tablet schema to schema cache if it does not have virtual columns.
if (cached_schema == nullptr && !schema_key.empty() &&
- tablet_schema->num_virtual_columns() == 0) {
+ tablet_schema->num_virtual_columns() == 0 && !tablet_schema->has_pruned_columns()) {
SchemaCache::instance()->insert_schema(schema_key, tablet_schema);
}
@@ -543,6 +545,22 @@ Status OlapScanner::_init_return_columns() {
_vir_col_idx_to_type[idx_in_block]->get_name());
}
+ const auto& column = tablet_schema->column(index);
+ if (!slot->all_access_paths().empty()) {
+ _tablet_reader_params.all_access_paths.insert(
+ {column.unique_id(), slot->all_access_paths()});
+ }
+
+ if (!slot->predicate_access_paths().empty()) {
+ _tablet_reader_params.predicate_access_paths.insert(
+ {column.unique_id(), slot->predicate_access_paths()});
+ }
+
+ if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT &&
+ !slot->all_access_paths().empty()) {
+ tablet_schema->add_pruned_columns_data_type(column.unique_id(), slot->type());
+ }
+
_return_columns.push_back(index);
if (slot->is_nullable() && !tablet_schema->column(index).is_nullable()) {
_tablet_columns_convert_to_null_set.emplace(index);
diff --git a/be/test/olap/rowset/segment_v2/column_reader_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
new file mode 100644
index 00000000000..98478df7016
--- /dev/null
+++ b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include "olap/rowset/segment_v2/column_reader.h"
+
+#include <gen_cpp/Descriptors_types.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include "agent/be_exec_version_manager.h"
+#include "common/config.h"
+#include "gen_cpp/olap_file.pb.h"
+#include "gen_cpp/segment_v2.pb.h"
+#include "io/fs/file_reader.h"
+#include "mock/mock_segment.h"
+#include "olap/rowset/segment_v2/column_reader_cache.h"
+#include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/variant/variant_column_reader.h"
+#include "olap/tablet_schema.h"
+#include "vec/json/path_in_data.h"
+
+namespace doris::segment_v2 {
+class ColumnReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {}
+ void TearDown() override {}
+};
+
+TEST_F(ColumnReaderTest, StructAccessPaths) {
+ auto create_struct_iterator = []() {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+ auto sub_reader2 = std::make_shared<ColumnReader>();
+ auto sub_iterator2 = std::make_unique<FileColumnIterator>(sub_reader2);
+ sub_iterator2->set_column_name("sub_col_2");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(sub_iterator2));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto iterator = create_struct_iterator();
+ auto st = iterator->set_access_paths(TColumnAccessPaths {}, TColumnAccessPaths {});
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NORMAL_READING);
+
+ TColumnAccessPaths all_access_paths;
+ all_access_paths.emplace_back();
+
+ TColumnAccessPaths predicate_access_paths;
+ predicate_access_paths.emplace_back();
+
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ // empty paths leads to error
+ ASSERT_FALSE(st.ok());
+
+ // Only reading sub_col_1
+ // sub_col_2 should be set to SKIP_READING
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
+
+ predicate_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
+
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ // invalid name leads to error
+ ASSERT_FALSE(st.ok());
+
+ iterator->set_column_name("self");
+ // now column name is "self", should be ok
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::SKIP_READING);
+
+ // Reading all sub columns
+ all_access_paths[0].data_access_path.path = {"self"};
+ iterator = create_struct_iterator();
+ iterator->set_column_name("self");
+ st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::READING_FOR_PREDICATE);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::NEED_TO_READ);
+}
+
+TEST_F(ColumnReaderTest, MultiAccessPaths) {
+ auto create_struct_iterator = []() {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+ auto sub_reader2 = std::make_shared<ColumnReader>();
+ auto sub_iterator2 = std::make_unique<FileColumnIterator>(sub_reader2);
+ sub_iterator2->set_column_name("sub_col_2");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(sub_iterator2));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto create_struct_iterator2 = [](ColumnIteratorUPtr&& nested_iterator) {
+ auto null_reader = std::make_shared<ColumnReader>();
+ auto null_iterator = std::make_unique<FileColumnIterator>(null_reader);
+
+ std::vector<ColumnIteratorUPtr> sub_column_iterators;
+ auto sub_reader1 = std::make_shared<ColumnReader>();
+ auto sub_iterator1 = std::make_unique<FileColumnIterator>(sub_reader1);
+ sub_iterator1->set_column_name("sub_col_1");
+
+ sub_column_iterators.emplace_back(std::move(sub_iterator1));
+ sub_column_iterators.emplace_back(std::move(nested_iterator));
+ auto iterator = std::make_unique<StructFileColumnIterator>(std::make_shared<ColumnReader>(),
+ std::move(null_iterator),
+ std::move(sub_column_iterators));
+ return iterator;
+ };
+
+ auto struct_iterator = create_struct_iterator();
+ struct_iterator->set_column_name("struct");
+
+ auto map_iterator = std::make_unique<MapFileColumnIterator>(
+ std::make_shared<ColumnReader>(),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()), // null iterator
+ std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>())),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()),
+ std::move(struct_iterator));
+
+ auto array_iterator = std::make_unique<ArrayFileColumnIterator>(
+ std::make_shared<ColumnReader>(),
+ std::make_unique<OffsetFileColumnIterator>(
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>())),
+ std::move(map_iterator),
+ std::make_unique<FileColumnIterator>(std::make_shared<ColumnReader>()));
+
+ // here create:
+ // struct<
+ // sub_col_1,
+ // sub_col_2: array<
+ // map<
+ // key,
+ // value: struct<
+ // sub_col_1,
+ // sub_col_2
+ // >
+ // >
+ // >
+ // >
+ array_iterator->set_column_name("sub_col_2");
+ auto iterator = create_struct_iterator2(std::move(array_iterator));
+ TColumnAccessPaths all_access_paths;
+ all_access_paths.emplace_back();
+
+ // all access paths:
+ // self.sub_col_2.*.KEYS
+ // predicates paths empty
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_2", "*", "KEYS"};
+
+ TColumnAccessPaths predicate_access_paths;
+
+ iterator->set_column_name("self");
+ auto st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
+
+ ASSERT_TRUE(st.ok()) << "failed to set access paths: " << st.to_string();
+ ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ ASSERT_EQ(iterator->_sub_column_iterators[0]->_reading_flag,
+ ColumnIterator::ReadingFlag::SKIP_READING);
+ ASSERT_EQ(iterator->_sub_column_iterators[1]->_reading_flag,
+ ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ auto* array_iter =
+ static_cast<ArrayFileColumnIterator*>(iterator->_sub_column_iterators[1].get());
+ ASSERT_EQ(array_iter->_item_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+
+ auto* map_iter = static_cast<MapFileColumnIterator*>(array_iter->_item_iterator.get());
+ ASSERT_EQ(map_iter->_key_iterator->_reading_flag, ColumnIterator::ReadingFlag::NEED_TO_READ);
+ ASSERT_EQ(map_iter->_val_iterator->_reading_flag, ColumnIterator::ReadingFlag::SKIP_READING);
+}
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java
new file mode 100644
index 00000000000..3df7baada4d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/TupleDescriptor.java
+// and modified by Doris
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.thrift.TColumnAccessPath;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
+
+/** AccessPathInfo */
+@Data
+@AllArgsConstructor
+public class AccessPathInfo {
+ private DataType prunedType;
+ private List<TColumnAccessPath> allAccessPaths;
+ private List<TColumnAccessPath> predicateAccessPaths;
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
index 87843f721c9..c0b35bd8f96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ColumnStats;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Type;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TSlotDescriptor;
import com.google.common.base.MoreObjects;
@@ -78,6 +79,10 @@ public class SlotDescriptor {
private boolean needMaterialize = true;
private boolean isAutoInc = false;
private Expr virtualColumn = null;
+ private List<TColumnAccessPath> allAccessPaths;
+ private List<TColumnAccessPath> predicateAccessPaths;
+ private List<TColumnAccessPath> displayAllAccessPaths;
+ private List<TColumnAccessPath> displayPredicateAccessPaths;
public SlotDescriptor(SlotId id, TupleDescriptor parent) {
@@ -132,6 +137,38 @@ public class SlotDescriptor {
return this.subColPath;
}
+ public List<TColumnAccessPath> getAllAccessPaths() {
+ return allAccessPaths;
+ }
+
+ public void setAllAccessPaths(List<TColumnAccessPath> allAccessPaths) {
+ this.allAccessPaths = allAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getPredicateAccessPaths() {
+ return predicateAccessPaths;
+ }
+
+ public void setPredicateAccessPaths(List<TColumnAccessPath> predicateAccessPaths) {
+ this.predicateAccessPaths = predicateAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getDisplayAllAccessPaths() {
+ return displayAllAccessPaths;
+ }
+
+ public void setDisplayAllAccessPaths(List<TColumnAccessPath> displayAllAccessPaths) {
+ this.displayAllAccessPaths = displayAllAccessPaths;
+ }
+
+ public List<TColumnAccessPath> getDisplayPredicateAccessPaths() {
+ return displayPredicateAccessPaths;
+ }
+
+ public void setDisplayPredicateAccessPaths(List<TColumnAccessPath> displayPredicateAccessPaths) {
+ this.displayPredicateAccessPaths = displayPredicateAccessPaths;
+ }
+
public TupleDescriptor getParent() {
return parent;
}
@@ -339,6 +376,12 @@ public class SlotDescriptor {
if (virtualColumn != null) {
tSlotDescriptor.setVirtualColumnExpr(virtualColumn.treeToThrift());
}
+ if (allAccessPaths != null) {
+ tSlotDescriptor.setAllAccessPaths(allAccessPaths);
+ }
+ if (predicateAccessPaths != null) {
+ tSlotDescriptor.setPredicateAccessPaths(predicateAccessPaths);
+ }
return tSlotDescriptor;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index 8578b094897..55564d89f04 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -52,6 +52,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -185,6 +186,8 @@ public abstract class FileScanNode extends ExternalScanNode {
}
output.append(String.format("numNodes=%s", numNodes)).append("\n");
+ printNestedColumns(output, prefix);
+
// pushdown agg
output.append(prefix).append(String.format("pushdown agg=%s", pushDownAggNoGroupingOp));
if (pushDownAggNoGroupingOp.equals(TPushAggOp.COUNT)) {
@@ -210,7 +213,10 @@ public abstract class FileScanNode extends ExternalScanNode {
TExpr tExpr = new TExpr();
tExpr.setNodes(Lists.newArrayList());
+ int index = 0;
+ List<SlotDescriptor> slots = desc.getSlots();
for (Column column : getColumns()) {
+ SlotDescriptor slotDescriptor = slots.get(index++);
Expr expr;
Expression expression;
if (column.getDefaultValue() != null) {
@@ -225,7 +231,9 @@ public abstract class FileScanNode extends ExternalScanNode {
if (useVarcharAsNull) {
expression = new NullLiteral(VarcharType.SYSTEM_DEFAULT);
} else {
- expression = new NullLiteral(DataType.fromCatalogType(column.getType()));
+ // the nested type(map/array/struct) maybe pruned,
+ // we should use the pruned type to avoid be core
+ expression = new NullLiteral(DataType.fromCatalogType(slotDescriptor.getType()));
}
} else {
expression = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 0a841e14dad..38fa6aedb91 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -286,6 +286,8 @@ public class StatementContext implements Closeable {
private Optional<Map<TableIf, Set<Expression>>> mvRefreshPredicates = Optional.empty();
+ private boolean hasNestedColumns;
+
public StatementContext() {
this(ConnectContext.get(), null, 0);
}
@@ -1026,4 +1028,12 @@ public class StatementContext implements Closeable {
public void setSkipPrunePredicate(boolean skipPrunePredicate) {
this.skipPrunePredicate = skipPrunePredicate;
}
+
+ public boolean hasNestedColumns() {
+ return hasNestedColumns;
+ }
+
+ public void setHasNestedColumns(boolean hasNestedColumns) {
+ this.hasNestedColumns = hasNestedColumns;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 44186fe3711..6b14de78e00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -837,10 +837,11 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
OlapTable olapTable = olapScan.getTable();
// generate real output tuple
TupleDescriptor tupleDescriptor = generateTupleDesc(slots, olapTable, context);
+ List<SlotDescriptor> slotDescriptors = tupleDescriptor.getSlots();
// put virtual column expr into slot desc
Map<ExprId, Expression> slotToVirtualColumnMap = olapScan.getSlotToVirtualColumnMap();
- for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) {
+ for (SlotDescriptor slotDescriptor : slotDescriptors) {
ExprId exprId = context.findExprId(slotDescriptor.getId());
if (slotToVirtualColumnMap.containsKey(exprId)) {
slotDescriptor.setVirtualColumn(ExpressionTranslator.translate(
@@ -910,7 +911,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
&& !isComplexDataType(slot.getDataType())
&& !StatisticConstants.isSystemTable(olapTable)
&& !inVisibleCol) {
- context.addUnknownStatsColumn(olapScanNode, tupleDescriptor.getSlots().get(i).getId());
+ context.addUnknownStatsColumn(olapScanNode, slotDescriptors.get(i).getId());
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index 7553a81d5b0..cbbd425c5b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -29,6 +29,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.processor.post.TopnFilterContext;
import org.apache.doris.nereids.processor.post.runtimefilterv2.RuntimeFilterContextV2;
import org.apache.doris.nereids.trees.expressions.CTEId;
@@ -67,6 +68,7 @@ import javax.annotation.Nullable;
*/
public class PlanTranslatorContext {
private final ConnectContext connectContext;
+ private final StatementContext statementContext;
private final List<PlanFragment> planFragments = Lists.newArrayList();
private DescriptorTable descTable;
@@ -119,16 +121,20 @@ public class PlanTranslatorContext {
private final Set<SlotId> virtualColumnIds = Sets.newHashSet();
+ /** PlanTranslatorContext */
public PlanTranslatorContext(CascadesContext ctx) {
this.connectContext = ctx.getConnectContext();
+ this.statementContext = ctx.getStatementContext();
this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext());
this.topnFilterContext = ctx.getTopnFilterContext();
this.runtimeFilterV2Context = ctx.getRuntimeFilterV2Context();
this.descTable = new DescriptorTable();
}
+ /** PlanTranslatorContext */
public PlanTranslatorContext(CascadesContext ctx, DescriptorTable descTable) {
this.connectContext = ctx.getConnectContext();
+ this.statementContext = ctx.getStatementContext();
this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext());
this.topnFilterContext = ctx.getTopnFilterContext();
this.runtimeFilterV2Context = ctx.getRuntimeFilterV2Context();
@@ -141,6 +147,7 @@ public class PlanTranslatorContext {
@VisibleForTesting
public PlanTranslatorContext() {
this.connectContext = null;
+ this.statementContext = new StatementContext();
this.translator = null;
this.topnFilterContext = new TopnFilterContext();
IdGenerator<RuntimeFilterId> runtimeFilterIdGen = RuntimeFilterId.createGenerator();
@@ -180,6 +187,10 @@ public class PlanTranslatorContext {
return connectContext;
}
+ public StatementContext getStatementContext() {
+ return statementContext;
+ }
+
public Set<ScanNode> getScanNodeWithUnknownColumnStats() {
return statsUnknownColumnsMap.keySet();
}
@@ -329,6 +340,13 @@ public class PlanTranslatorContext {
}
this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef);
slotDescriptor.setIsNullable(slotReference.nullable());
+
+ if (slotReference.getAllAccessPaths().isPresent()) {
+ slotDescriptor.setAllAccessPaths(slotReference.getAllAccessPaths().get());
+ slotDescriptor.setPredicateAccessPaths(slotReference.getPredicateAccessPaths().get());
+ slotDescriptor.setDisplayAllAccessPaths(slotReference.getDisplayAllAccessPaths().get());
+ slotDescriptor.setDisplayPredicateAccessPaths(slotReference.getDisplayPredicateAccessPaths().get());
+ }
return slotDescriptor;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 29f0b0c79a8..07339e6dd63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -109,6 +109,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeProjectable;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperationsExcept;
import org.apache.doris.nereids.rules.rewrite.MergeTopNs;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning;
import org.apache.doris.nereids.rules.rewrite.NormalizeSort;
import org.apache.doris.nereids.rules.rewrite.OperativeColumnDerive;
import org.apache.doris.nereids.rules.rewrite.OrExpansion;
@@ -913,6 +914,11 @@ public class Rewriter extends AbstractBatchJobExecutor {
)
));
}
+ rewriteJobs.add(
+ topic("nested column prune",
+ custom(RuleType.NESTED_COLUMN_PRUNING, NestedColumnPruning::new)
+ )
+ );
rewriteJobs.addAll(jobs(
topic("rewrite cte sub-tree after sub path push down",
custom(RuleType.CLEAR_CONTEXT_STATUS, ClearContextStatus::new),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 7e3e710eee2..a2268581a47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -115,6 +115,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownNestedColumn;
import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
@@ -166,6 +167,7 @@ public class RuleSet {
new PushDownFilterThroughSetOperation(),
new PushDownFilterThroughGenerate(),
new PushDownProjectThroughLimit(),
+ new PushDownNestedColumn(),
new EliminateOuterJoin(),
new MergeProjectable(),
new MergeFilters(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 7b32bb710e0..70bedd1b85b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -195,6 +195,12 @@ public enum RuleType {
PUSH_DOWN_PREDICATE_THROUGH_AGGREGATION(RuleTypeClass.REWRITE),
PUSH_DOWN_PREDICATE_THROUGH_REPEAT(RuleTypeClass.REWRITE),
PUSH_DOWN_EXPRESSIONS_IN_HASH_CONDITIONS(RuleTypeClass.REWRITE),
+ // Pushdown nested column project
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N(RuleTypeClass.REWRITE),
// Pushdown filter
PUSH_DOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSH_DOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE),
@@ -219,6 +225,7 @@ public enum RuleType {
ADD_PROJECT_FOR_JOIN(RuleTypeClass.REWRITE),
ADD_PROJECT_FOR_UNIQUE_FUNCTION(RuleTypeClass.REWRITE),
VARIANT_SUB_PATH_PRUNING(RuleTypeClass.REWRITE),
+ NESTED_COLUMN_PRUNING(RuleTypeClass.REWRITE),
CLEAR_CONTEXT_STATUS(RuleTypeClass.REWRITE),
COLUMN_PRUNING(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 84652b9eed0..36b7f0064ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -424,7 +424,8 @@ public class BindRelation extends OneAnalysisRuleFactory {
if (hmsTable.getDlaType() == DLAType.HUDI) {
LogicalHudiScan hudiScan = new LogicalHudiScan(unboundRelation.getRelationId(), hmsTable,
qualifierWithoutTableName, ImmutableList.of(), Optional.empty(),
- unboundRelation.getTableSample(), unboundRelation.getTableSnapshot());
+ unboundRelation.getTableSample(), unboundRelation.getTableSnapshot(),
+ Optional.empty());
hudiScan = hudiScan.withScanParams(
hmsTable, Optional.ofNullable(unboundRelation.getScanParams()));
return hudiScan;
@@ -434,7 +435,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
}
case ICEBERG_EXTERNAL_TABLE:
IcebergExternalTable icebergExternalTable = (IcebergExternalTable) table;
@@ -464,7 +465,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
qualifierWithoutTableName, ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
case PAIMON_EXTERNAL_TABLE:
case MAX_COMPUTE_EXTERNAL_TABLE:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
@@ -473,7 +474,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
qualifierWithoutTableName, ImmutableList.of(),
unboundRelation.getTableSample(),
unboundRelation.getTableSnapshot(),
- Optional.ofNullable(unboundRelation.getScanParams()));
+ Optional.ofNullable(unboundRelation.getScanParams()), Optional.empty());
case SCHEMA:
LogicalSchemaScan schemaScan = new LogicalSchemaScan(unboundRelation.getRelationId(), table,
qualifierWithoutTableName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
index 35adb692be3..b90dc3adc81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java
@@ -93,6 +93,7 @@ import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -285,6 +286,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
}
outerScope.get().getCorrelatedSlots().add((Slot) firstBound);
}
+ if (firstBound.getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
return firstBound;
default:
if (enableExactMatch) {
@@ -304,6 +308,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
.filter(bound -> unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1)
.collect(Collectors.toList());
if (exactMatch.size() == 1) {
+ if (exactMatch.get(0).getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
return exactMatch.get(0);
}
}
@@ -340,6 +347,9 @@ public class ExpressionAnalyzer extends SubExprAnalyzer<ExpressionRewriteContext
if (!(slot instanceof SlotReference) || (((SlotReference) slot).isVisible()) || showHidden) {
showSlots.add(slot);
}
+ if (slot.getDataType() instanceof NestedColumnPrunable) {
+ context.cascadesContext.getStatementContext().setHasNestedColumns(true);
+ }
}
ImmutableList<Slot> slots = showSlots.build();
switch (qualifier.size()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
index 2c957912ce9..c4302798e25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.rules.expression.rules.LogToLn;
import org.apache.doris.nereids.rules.expression.rules.MedianConvert;
import org.apache.doris.nereids.rules.expression.rules.MergeDateTrunc;
import org.apache.doris.nereids.rules.expression.rules.NormalizeBinaryPredicatesRule;
+import org.apache.doris.nereids.rules.expression.rules.NormalizeStructElement;
import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticComparisonRule;
import org.apache.doris.nereids.rules.expression.rules.SimplifyArithmeticRule;
import org.apache.doris.nereids.rules.expression.rules.SimplifyCastRule;
@@ -68,6 +69,7 @@ public class ExpressionNormalization extends ExpressionRewrite {
SimplifyArithmeticComparisonRule.INSTANCE,
ConvertAggStateCast.INSTANCE,
MergeDateTrunc.INSTANCE,
+ NormalizeStructElement.INSTANCE,
CheckCast.INSTANCE,
SimplifyEqualBooleanLiteral.INSTANCE
)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
index f0d1b93ed12..0265b7bf381 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
@@ -62,6 +62,7 @@ public enum ExpressionRuleType {
SIMPLIFY_RANGE,
SIMPLIFY_SELF_COMPARISON,
SUPPORT_JAVA_DATE_FORMATTER,
+ NORMALIZE_STRUCT_ELEMENT,
TOPN_TO_MAX;
public int type() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java
new file mode 100644
index 00000000000..dc50921bc1e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * given column s has data type: struct<a: int, b: int>, if exists struct_element(s, 2), we will rewrite
+ * to struct_element(s, 'b')
+ */
+public class NormalizeStructElement implements ExpressionPatternRuleFactory {
+ public static final NormalizeStructElement INSTANCE = new NormalizeStructElement();
+
+ @Override
+ public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+ return ImmutableList.of(
+ matchesType(StructElement.class).then(NormalizeStructElement::normalize)
+ .toRule(ExpressionRuleType.NORMALIZE_STRUCT_ELEMENT)
+ );
+ }
+
+ private static StructElement normalize(StructElement structElement) {
+ Expression field = structElement.getArgument(1);
+ if (field instanceof IntegerLikeLiteral) {
+ int fieldIndex = ((Number) ((IntegerLikeLiteral) field).getValue()).intValue();
+ StructType structType = (StructType) structElement.getArgument(0).getDataType();
+ List<StructField> fields = structType.getFields();
+ if (fieldIndex >= 0 && fieldIndex <= fields.size()) {
+ return structElement.withChildren(
+ ImmutableList.of(
+ structElement.child(0),
+ new StringLiteral(fields.get(fieldIndex - 1).getName())
+ )
+ );
+ }
+ }
+ return structElement;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
new file mode 100644
index 00000000000..4fe9e9e2f74
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -0,0 +1,511 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectorContext;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference.ArrayItemSlot;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirst;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAny;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayReverseSplit;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapKeys;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MapValues;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TAccessPathType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Stack;
+
+/**
+ * collect the access path, for example: `select struct_element(s, 'data')` has access path: ['s', 'data']
+ */
+public class AccessPathExpressionCollector extends DefaultExpressionVisitor<Void, CollectorContext> {
+ private StatementContext statementContext;
+ private boolean bottomPredicate;
+ private Multimap<Integer, CollectAccessPathResult> slotToAccessPaths;
+ private Stack<Map<String, Expression>> nameToLambdaArguments = new Stack<>();
+
+ public AccessPathExpressionCollector(
+ StatementContext statementContext, Multimap<Integer, CollectAccessPathResult> slotToAccessPaths,
+ boolean bottomPredicate) {
+ this.statementContext = statementContext;
+ this.slotToAccessPaths = slotToAccessPaths;
+ this.bottomPredicate = bottomPredicate;
+ }
+
+ public void collect(Expression expression) {
+ expression.accept(this, new CollectorContext(statementContext, bottomPredicate));
+ }
+
+ private Void continueCollectAccessPath(Expression expr, CollectorContext context) {
+ return expr.accept(this, context);
+ }
+
+ @Override
+ public Void visit(Expression expr, CollectorContext context) {
+ for (Expression child : expr.children()) {
+ child.accept(this, new CollectorContext(context.statementContext, context.bottomFilter));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSlotReference(SlotReference slotReference, CollectorContext context) {
+ DataType dataType = slotReference.getDataType();
+ if (dataType instanceof NestedColumnPrunable) {
+ context.accessPathBuilder.addPrefix(slotReference.getName().toLowerCase());
+ ImmutableList<String> path = Utils.fastToImmutableList(context.accessPathBuilder.accessPath);
+ int slotId = slotReference.getExprId().asInt();
+ slotToAccessPaths.put(slotId, new CollectAccessPathResult(path, context.bottomFilter, context.type));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitArrayItemSlot(ArrayItemSlot arrayItemSlot, CollectorContext context) {
+ if (nameToLambdaArguments.isEmpty()) {
+ return null;
+ }
+ context.accessPathBuilder.addPrefix("*");
+ Expression argument = nameToLambdaArguments.peek().get(arrayItemSlot.getName());
+ if (argument == null) {
+ return null;
+ }
+ return continueCollectAccessPath(argument, context);
+ }
+
+ @Override
+ public Void visitAlias(Alias alias, CollectorContext context) {
+ return alias.child(0).accept(this, context);
+ }
+
+ @Override
+ public Void visitCast(Cast cast, CollectorContext context) {
+ if (!context.accessPathBuilder.isEmpty()
+ && cast.getDataType() instanceof NestedColumnPrunable
+ && cast.child().getDataType() instanceof NestedColumnPrunable) {
+
+ DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
+
+ List<String> replacePath = new ArrayList<>(context.accessPathBuilder.getPathList());
+ if (originTree.replacePathByAnotherTree(castTree, replacePath, 0)) {
+ CollectorContext castContext = new CollectorContext(context.statementContext, context.bottomFilter);
+ castContext.accessPathBuilder.accessPath.addAll(replacePath);
+ return continueCollectAccessPath(cast.child(), castContext);
+ }
+ }
+ return cast.child(0).accept(this,
+ new CollectorContext(context.statementContext, context.bottomFilter)
+ );
+ }
+
+ // array element at
+ @Override
+ public Void visitElementAt(ElementAt elementAt, CollectorContext context) {
+ List<Expression> arguments = elementAt.getArguments();
+ Expression first = arguments.get(0);
+ if (first.getDataType().isArrayType() || first.getDataType().isMapType()) {
+ context.accessPathBuilder.addPrefix("*");
+ continueCollectAccessPath(first, context);
+
+ for (int i = 1; i < arguments.size(); i++) {
+ visit(arguments.get(i), context);
+ }
+ return null;
+ } else {
+ return visit(elementAt, context);
+ }
+ }
+
+ // struct element_at
+ @Override
+ public Void visitStructElement(StructElement structElement, CollectorContext context) {
+ List<Expression> arguments = structElement.getArguments();
+ Expression struct = arguments.get(0);
+ Expression fieldName = arguments.get(1);
+ DataType fieldType = fieldName.getDataType();
+
+ if (fieldName.isLiteral() && (fieldType.isIntegerLikeType() || fieldType.isStringLikeType())) {
+ if (fieldType.isIntegerLikeType()) {
+ int fieldIndex = ((Number) ((Literal) fieldName).getValue()).intValue();
+ List<StructField> fields = ((StructType) struct.getDataType()).getFields();
+ if (fieldIndex >= 1 && fieldIndex <= fields.size()) {
+ String realFieldName = fields.get(fieldIndex - 1).getName();
+ context.accessPathBuilder.addPrefix(realFieldName);
+ return continueCollectAccessPath(struct, context);
+ }
+ }
+ context.accessPathBuilder.addPrefix(((Literal) fieldName).getStringValue().toLowerCase());
+ return continueCollectAccessPath(struct, context);
+ }
+
+ for (Expression argument : arguments) {
+ visit(argument, context);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitMapKeys(MapKeys mapKeys, CollectorContext context) {
+ context = new CollectorContext(context.statementContext, context.bottomFilter);
+ context.accessPathBuilder.addPrefix("KEYS");
+ return continueCollectAccessPath(mapKeys.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapValues(MapValues mapValues, CollectorContext context) {
+ LinkedList<String> suffixPath = context.accessPathBuilder.accessPath;
+ if (!suffixPath.isEmpty() && suffixPath.get(0).equals("*")) {
+ CollectorContext removeStarContext
+ = new CollectorContext(context.statementContext, context.bottomFilter);
+ removeStarContext.accessPathBuilder.accessPath.addAll(suffixPath.subList(1, suffixPath.size()));
+ removeStarContext.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapValues.getArgument(0), removeStarContext);
+ }
+ context.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapValues.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsKey(MapContainsKey mapContainsKey, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("KEYS");
+ return continueCollectAccessPath(mapContainsKey.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsValue(MapContainsValue mapContainsValue, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("VALUES");
+ return continueCollectAccessPath(mapContainsValue.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitMapContainsEntry(MapContainsEntry mapContainsEntry, CollectorContext context) {
+ context.accessPathBuilder.addPrefix("*");
+ return continueCollectAccessPath(mapContainsEntry.getArgument(0), context);
+ }
+
+ @Override
+ public Void visitArrayMap(ArrayMap arrayMap, CollectorContext context) {
+ // ARRAY_MAP(lambda, <arr> [ , <arr> ... ] )
+
+ Expression argument = arrayMap.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMap, context);
+ }
+
+ @Override
+ public Void visitArrayCount(ArrayCount arrayCount, CollectorContext context) {
+ // ARRAY_COUNT(<lambda>, <arr>[, ... ])
+
+ Expression argument = arrayCount.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayCount, context);
+ }
+
+ @Override
+ public Void visitArrayExists(ArrayExists arrayExists, CollectorContext context) {
+ // ARRAY_EXISTS([ <lambda>, ] <arr1> [, <arr2> , ...] )
+
+ Expression argument = arrayExists.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ return collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayExists, context);
+ }
+
+ @Override
+ public Void visitArrayFilter(ArrayFilter arrayFilter, CollectorContext context) {
+ // ARRAY_FILTER(<lambda>, <arr>)
+
+ Expression argument = arrayFilter.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFilter, context);
+ }
+
+ @Override
+ public Void visitArrayFirst(ArrayFirst arrayFirst, CollectorContext context) {
+ // ARRAY_FIRST(<lambda>, <arr>)
+
+ Expression argument = arrayFirst.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFirst, context);
+ }
+
+ @Override
+ public Void visitArrayFirstIndex(ArrayFirstIndex arrayFirstIndex, CollectorContext context) {
+ // ARRAY_FIRST_INDEX(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayFirstIndex.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayFirstIndex, context);
+ }
+
+ @Override
+ public Void visitArrayLast(ArrayLast arrayLast, CollectorContext context) {
+ // ARRAY_LAST(<lambda>, <arr>)
+
+ Expression argument = arrayLast.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayLast, context);
+ }
+
+ @Override
+ public Void visitArrayLastIndex(ArrayLastIndex arrayLastIndex, CollectorContext context) {
+ // ARRAY_LAST_INDEX(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayLastIndex.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayLastIndex, context);
+ }
+
+ @Override
+ public Void visitArrayMatchAny(ArrayMatchAny arrayMatchAny, CollectorContext context) {
+ // array_match_any(lambda, <arr> [, <arr> ...])
+
+ Expression argument = arrayMatchAny.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMatchAny, context);
+ }
+
+ @Override
+ public Void visitArrayMatchAll(ArrayMatchAll arrayMatchAll, CollectorContext context) {
+ // array_match_all(lambda, <arr> [, <arr> ...])
+
+ Expression argument = arrayMatchAll.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayMatchAll, context);
+ }
+
+ @Override
+ public Void visitArrayReverseSplit(ArrayReverseSplit arrayReverseSplit, CollectorContext context) {
+ // ARRAY_REVERSE_SPLIT(<lambda>, <arr> [, ...])
+
+ Expression argument = arrayReverseSplit.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arrayReverseSplit, context);
+ }
+
+ @Override
+ public Void visitArraySplit(ArraySplit arraySplit, CollectorContext context) {
+ // ARRAY_SPLIT(<lambda>, arr [, ...])
+
+ Expression argument = arraySplit.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arraySplit, context);
+ }
+
+ @Override
+ public Void visitArraySortBy(ArraySortBy arraySortBy, CollectorContext context) {
+ // ARRAY_SORTBY(<lambda>, <arr> [, ...])
+
+ Expression argument = arraySortBy.getArgument(0);
+ if ((argument instanceof Lambda)) {
+ collectArrayPathInLambda((Lambda) argument, context);
+ }
+ return visit(arraySortBy, context);
+ }
+
+ // @Override
+ // public Void visitIsNull(IsNull isNull, CollectorContext context) {
+ // if (context.accessPathBuilder.isEmpty()) {
+ // context.setType(TAccessPathType.META);
+ // return continueCollectAccessPath(isNull.child(), context);
+ // }
+ // return visit(isNull, context);
+ // }
+
+ private Void collectArrayPathInLambda(Lambda lambda, CollectorContext context) {
+ List<Expression> arguments = lambda.getArguments();
+ Map<String, Expression> nameToArray = Maps.newLinkedHashMap();
+ for (Expression argument : arguments) {
+ if (argument instanceof ArrayItemReference) {
+ nameToArray.put(((ArrayItemReference) argument).getName(), argument.child(0));
+ }
+ }
+
+ List<String> path = context.accessPathBuilder.getPathList();
+ if (!path.isEmpty() && path.get(0).equals("*")) {
+ context.accessPathBuilder.removePrefix();
+ }
+
+ nameToLambdaArguments.push(nameToArray);
+ try {
+ continueCollectAccessPath(arguments.get(0), context);
+ } finally {
+ nameToLambdaArguments.pop();
+ }
+ return null;
+ }
+
+ /** CollectorContext */
+ public static class CollectorContext {
+ private StatementContext statementContext;
+ private AccessPathBuilder accessPathBuilder;
+ private boolean bottomFilter;
+ private TAccessPathType type;
+
+ public CollectorContext(StatementContext statementContext, boolean bottomFilter) {
+ this.statementContext = statementContext;
+ this.accessPathBuilder = new AccessPathBuilder();
+ this.bottomFilter = bottomFilter;
+ this.type = TAccessPathType.DATA;
+ }
+
+ public TAccessPathType getType() {
+ return type;
+ }
+
+ public void setType(TAccessPathType type) {
+ this.type = type;
+ }
+ }
+
+ private static class AccessPathBuilder {
+ private LinkedList<String> accessPath;
+
+ public AccessPathBuilder() {
+ accessPath = new LinkedList<>();
+ }
+
+ public AccessPathBuilder addPrefix(String prefix) {
+ accessPath.addFirst(prefix);
+ return this;
+ }
+
+ public AccessPathBuilder removePrefix() {
+ accessPath.removeFirst();
+ return this;
+ }
+
+ public List<String> getPathList() {
+ return accessPath;
+ }
+
+ public boolean isEmpty() {
+ return accessPath.isEmpty();
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", accessPath);
+ }
+ }
+
+ /** AccessPathIsPredicate */
+ public static class CollectAccessPathResult {
+ private final List<String> path;
+ private final boolean isPredicate;
+ private final TAccessPathType type;
+
+ public CollectAccessPathResult(List<String> path, boolean isPredicate, TAccessPathType type) {
+ this.path = path;
+ this.isPredicate = isPredicate;
+ this.type = type;
+ }
+
+ public TAccessPathType getType() {
+ return type;
+ }
+
+ public List<String> getPath() {
+ return path;
+ }
+
+ public boolean isPredicate() {
+ return isPredicate;
+ }
+
+ @Override
+ public String toString() {
+ return String.join(".", path) + ", " + isPredicate;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ CollectAccessPathResult that = (CollectAccessPathResult) o;
+ return isPredicate == that.isPredicate && Objects.equals(path, that.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return path.hashCode();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
new file mode 100644
index 00000000000..514f7bb1e8c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/** AccessPathPlanCollector */
+public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementContext> {
+ private Multimap<Integer, CollectAccessPathResult> allSlotToAccessPaths = LinkedHashMultimap.create();
+ private Map<Slot, List<CollectAccessPathResult>> scanSlotToAccessPaths = new LinkedHashMap<>();
+
+ public Map<Slot, List<CollectAccessPathResult>> collect(Plan root, StatementContext context) {
+ root.accept(this, context);
+ return scanSlotToAccessPaths;
+ }
+
+ @Override
+ public Void visitLogicalFilter(LogicalFilter<? extends Plan> filter, StatementContext context) {
+ boolean bottomFilter = filter.child().arity() == 0;
+ collectByExpressions(filter, context, bottomFilter);
+ return filter.child().accept(this, context);
+ }
+
+ @Override
+ public Void visitLogicalCTEAnchor(
+ LogicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor, StatementContext context) {
+ // first, collect access paths in the outer slots, and propagate outer slot's access path to inner slots
+ cteAnchor.right().accept(this, context);
+
+ // second, push down access path in the inner slots
+ cteAnchor.left().accept(this, context);
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, StatementContext context) {
+ // propagate outer slot's access path to inner slots
+ for (Entry<Slot, Slot> slots : cteConsumer.getConsumerToProducerOutputMap().entrySet()) {
+ Slot outerSlot = slots.getKey();
+
+ if (outerSlot.getDataType() instanceof NestedColumnPrunable) {
+ int outerSlotId = outerSlot.getExprId().asInt();
+ int innerSlotId = slots.getValue().getExprId().asInt();
+ allSlotToAccessPaths.putAll(innerSlotId, allSlotToAccessPaths.get(outerSlotId));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer, StatementContext context) {
+ return cteProducer.child().accept(this, context);
+ }
+
+ @Override
+ public Void visitLogicalUnion(LogicalUnion union, StatementContext context) {
+ // now we will not prune complex type through union, because we can not prune the complex type's literal,
+ // for example, we can not prune the literal now: array(map(1, named_struct('a', 100, 'b', 100))),
+ // so we can not prune this sql:
+ // select struct_element(map_values(s[0]), 'a')
+ // from (
+ // select s from tbl
+ // union all
+ // select array(map(1, named_struct('a', 100, 'b', 100)))
+ // ) tbl;
+ //
+ // so we will not propagate access paths through the union
+ for (Plan child : union.children()) {
+ child.accept(this, context);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalOlapScan(LogicalOlapScan olapScan, StatementContext context) {
+ for (Slot slot : olapScan.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalFileScan(LogicalFileScan fileScan, StatementContext context) {
+ for (Slot slot : fileScan.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, StatementContext context) {
+ for (Slot slot : tvfRelation.getOutput()) {
+ if (!(slot.getDataType() instanceof NestedColumnPrunable)) {
+ continue;
+ }
+ Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(slot.getExprId().asInt());
+ if (!accessPaths.isEmpty()) {
+ scanSlotToAccessPaths.put(slot, new ArrayList<>(accessPaths));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(Plan plan, StatementContext context) {
+ collectByExpressions(plan, context);
+
+ for (Plan child : plan.children()) {
+ child.accept(this, context);
+ }
+ return null;
+ }
+
+ private void collectByExpressions(Plan plan, StatementContext context) {
+ collectByExpressions(plan, context, false);
+ }
+
+ private void collectByExpressions(Plan plan, StatementContext context, boolean bottomPredicate) {
+ AccessPathExpressionCollector exprCollector
+ = new AccessPathExpressionCollector(context, allSlotToAccessPaths, bottomPredicate);
+ for (Expression expression : plan.getExpressions()) {
+ exprCollector.collect(expression);
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
new file mode 100644
index 00000000000..de22e9d2101
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -0,0 +1,487 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.AccessPathInfo;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.TreeMultimap;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+
+/**
+ * <li> 1. prune the data type of struct/map
+ *
+ * <p> for example, column s is a struct<id: int, value: double>,
+ * and `select struct(s, 'id') from tbl` will return the data type: `struct<id: int>`
+ * </p>
+ * </li>
+ *
+ * <li> 2. collect the access paths
+ * <p> for example, select struct(s, 'id'), struct(s, 'data') from tbl` will collect the access path:
+ * [s.id, s.data]
+ * </p>
+ * </li>
+ */
+public class NestedColumnPruning implements CustomRewriter {
+ public static final Logger LOG = LogManager.getLogger(NestedColumnPruning.class);
+
+ @Override
+ public Plan rewriteRoot(Plan plan, JobContext jobContext) {
+ try {
+ StatementContext statementContext = jobContext.getCascadesContext().getStatementContext();
+ SessionVariable sessionVariable = statementContext.getConnectContext().getSessionVariable();
+ if (!sessionVariable.enablePruneNestedColumns || !statementContext.hasNestedColumns()) {
+ return plan;
+ }
+
+ AccessPathPlanCollector collector = new AccessPathPlanCollector();
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths = collector.collect(plan, statementContext);
+ Map<Integer, AccessPathInfo> slotToResult = pruneDataType(slotToAccessPaths);
+
+ if (!slotToResult.isEmpty()) {
+ Map<Integer, AccessPathInfo> slotIdToPruneType = Maps.newLinkedHashMap();
+ for (Entry<Integer, AccessPathInfo> kv : slotToResult.entrySet()) {
+ Integer slotId = kv.getKey();
+ AccessPathInfo accessPathInfo = kv.getValue();
+ slotIdToPruneType.put(slotId, accessPathInfo);
+ }
+ SlotTypeReplacer typeReplacer = new SlotTypeReplacer(slotIdToPruneType);
+ return plan.accept(typeReplacer, null);
+ }
+ return plan;
+ } catch (Throwable t) {
+ LOG.warn("NestedColumnPruning failed.", t);
+ return plan;
+ }
+ }
+
+ private static Map<Integer, AccessPathInfo> pruneDataType(
+ Map<Slot, List<CollectAccessPathResult>> slotToAccessPaths) {
+ Map<Integer, AccessPathInfo> result = new LinkedHashMap<>();
+ Map<Slot, DataTypeAccessTree> slotIdToAllAccessTree = new LinkedHashMap<>();
+ Map<Slot, DataTypeAccessTree> slotIdToPredicateAccessTree = new LinkedHashMap<>();
+
+ Comparator<Pair<TAccessPathType, List<String>>> pathComparator = Comparator.comparing(
+ l -> StringUtils.join(l.second, "."));
+
+ Multimap<Integer, Pair<TAccessPathType, List<String>>> allAccessPaths = TreeMultimap.create(
+ Comparator.naturalOrder(), pathComparator);
+ Multimap<Integer, Pair<TAccessPathType, List<String>>> predicateAccessPaths = TreeMultimap.create(
+ Comparator.naturalOrder(), pathComparator);
+
+ // first: build access data type tree
+ for (Entry<Slot, List<CollectAccessPathResult>> kv : slotToAccessPaths.entrySet()) {
+ Slot slot = kv.getKey();
+ List<CollectAccessPathResult> collectAccessPathResults = kv.getValue();
+ for (CollectAccessPathResult collectAccessPathResult : collectAccessPathResults) {
+ List<String> path = collectAccessPathResult.getPath();
+ TAccessPathType pathType = collectAccessPathResult.getType();
+ DataTypeAccessTree allAccessTree = slotIdToAllAccessTree.computeIfAbsent(
+ slot, i -> DataTypeAccessTree.ofRoot(slot, pathType)
+ );
+ allAccessTree.setAccessByPath(path, 0, pathType);
+ allAccessPaths.put(slot.getExprId().asInt(), Pair.of(pathType, path));
+
+ if (collectAccessPathResult.isPredicate()) {
+ DataTypeAccessTree predicateAccessTree = slotIdToPredicateAccessTree.computeIfAbsent(
+ slot, i -> DataTypeAccessTree.ofRoot(slot, pathType)
+ );
+ predicateAccessTree.setAccessByPath(path, 0, pathType);
+ predicateAccessPaths.put(
+ slot.getExprId().asInt(), Pair.of(pathType, path)
+ );
+ }
+ }
+ }
+
+ // second: build non-predicate access paths
+ for (Entry<Slot, DataTypeAccessTree> kv : slotIdToAllAccessTree.entrySet()) {
+ Slot slot = kv.getKey();
+ DataTypeAccessTree accessTree = kv.getValue();
+ DataType prunedDataType = accessTree.pruneDataType().orElse(slot.getDataType());
+
+ List<TColumnAccessPath> allPaths = new ArrayList<>();
+ boolean accessWholeColumn = false;
+ TAccessPathType accessWholeColumnType = TAccessPathType.META;
+ for (Pair<TAccessPathType, List<String>> pathInfo : allAccessPaths.get(slot.getExprId().asInt())) {
+ if (pathInfo.first == TAccessPathType.DATA) {
+ TDataAccessPath dataAccessPath = new TDataAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.setDataAccessPath(dataAccessPath);
+ allPaths.add(accessPath);
+ } else {
+ TMetaAccessPath dataAccessPath = new TMetaAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.setMetaAccessPath(dataAccessPath);
+ allPaths.add(accessPath);
+ }
+ // only retain access the whole root
+ if (pathInfo.second.size() == 1) {
+ accessWholeColumn = true;
+ if (pathInfo.first == TAccessPathType.DATA) {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ } else {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ }
+ if (accessWholeColumn) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(accessWholeColumnType);
+ SlotReference slotReference = (SlotReference) kv.getKey();
+ String wholeColumnName = slotReference.getOriginalColumn().get().getName();
+ if (accessWholeColumnType == TAccessPathType.DATA) {
+ accessPath.setDataAccessPath(new TDataAccessPath(ImmutableList.of(wholeColumnName)));
+ } else {
+ accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName)));
+ }
+ allPaths = ImmutableList.of(accessPath);
+ }
+ result.put(slot.getExprId().asInt(), new AccessPathInfo(prunedDataType, allPaths, new ArrayList<>()));
+ }
+
+ // third: build predicate access path
+ for (Entry<Slot, DataTypeAccessTree> kv : slotIdToPredicateAccessTree.entrySet()) {
+ Slot slot = kv.getKey();
+
+ List<TColumnAccessPath> predicatePaths = new ArrayList<>();
+ boolean accessWholeColumn = false;
+ TAccessPathType accessWholeColumnType = TAccessPathType.META;
+ for (Pair<TAccessPathType, List<String>> pathInfo : predicateAccessPaths.get(slot.getExprId().asInt())) {
+ if (pathInfo == null) {
+ throw new AnalysisException("This is a bug, please report this");
+ }
+
+ if (pathInfo.first == TAccessPathType.DATA) {
+ TDataAccessPath dataAccessPath = new TDataAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.setDataAccessPath(dataAccessPath);
+ predicatePaths.add(accessPath);
+ } else {
+ TMetaAccessPath dataAccessPath = new TMetaAccessPath();
+ dataAccessPath.setPath(new ArrayList<>(pathInfo.second));
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.setMetaAccessPath(dataAccessPath);
+ predicatePaths.add(accessPath);
+ }
+ // only retain access the whole root
+ if (pathInfo.second.size() == 1) {
+ accessWholeColumn = true;
+ if (pathInfo.first == TAccessPathType.DATA) {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ } else {
+ accessWholeColumnType = TAccessPathType.DATA;
+ }
+ }
+
+ if (accessWholeColumn) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(accessWholeColumnType);
+ SlotReference slotReference = (SlotReference) kv.getKey();
+ String wholeColumnName = slotReference.getOriginalColumn().get().getName();
+ if (accessWholeColumnType == TAccessPathType.DATA) {
+ accessPath.setDataAccessPath(new TDataAccessPath(ImmutableList.of(wholeColumnName)));
+ } else {
+ accessPath.setMetaAccessPath(new TMetaAccessPath(ImmutableList.of(wholeColumnName)));
+ }
+ predicatePaths = ImmutableList.of(accessPath);
+ }
+
+ AccessPathInfo accessPathInfo = result.get(slot.getExprId().asInt());
+ accessPathInfo.getPredicateAccessPaths().addAll(predicatePaths);
+ }
+
+ return result;
+ }
+
+ /** DataTypeAccessTree */
+ public static class DataTypeAccessTree {
+ private DataType type;
+ private boolean isRoot;
+ private boolean accessPartialChild;
+ private boolean accessAll;
+ private TAccessPathType pathType;
+ private Map<String, DataTypeAccessTree> children = new LinkedHashMap<>();
+
+ public DataTypeAccessTree(DataType type, TAccessPathType pathType) {
+ this(false, type, pathType);
+ }
+
+ public DataTypeAccessTree(boolean isRoot, DataType type, TAccessPathType pathType) {
+ this.isRoot = isRoot;
+ this.type = type;
+ this.pathType = pathType;
+ }
+
+ /** pruneCastType */
+ public DataType pruneCastType(DataTypeAccessTree origin, DataTypeAccessTree cast) {
+ if (type instanceof StructType) {
+ Map<String, String> nameMapping = new LinkedHashMap<>();
+ List<String> castNames = new ArrayList<>(cast.children.keySet());
+ int i = 0;
+ for (String s : origin.children.keySet()) {
+ nameMapping.put(s, castNames.get(i++));
+ }
+ List<StructField> mappingFields = new ArrayList<>();
+ StructType originCastStructType = (StructType) cast.type;
+ for (Entry<String, DataTypeAccessTree> kv : children.entrySet()) {
+ String originName = kv.getKey();
+ String mappingName = nameMapping.getOrDefault(originName, originName);
+ DataTypeAccessTree originPrunedTree = kv.getValue();
+ DataType mappingType = originPrunedTree.pruneCastType(
+ origin.children.get(originName),
+ cast.children.get(mappingName)
+ );
+ StructField originCastField = originCastStructType.getField(mappingName);
+ mappingFields.add(
+ new StructField(mappingName, mappingType,
+ originCastField.isNullable(), originCastField.getComment())
+ );
+ }
+ return new StructType(mappingFields);
+ } else if (type instanceof ArrayType) {
+ return ArrayType.of(
+ children.values().iterator().next().pruneCastType(
+ origin.children.values().iterator().next(),
+ cast.children.values().iterator().next()
+ ),
+ ((ArrayType) cast.type).containsNull()
+ );
+ } else if (type instanceof MapType) {
+ return MapType.of(
+ children.get("KEYS").pruneCastType(origin.children.get("KEYS"), cast.children.get("KEYS")),
+ children.get("VALUES").pruneCastType(origin.children.get("VALUES"), cast.children.get("VALUES"))
+ );
+ } else {
+ return cast.type;
+ }
+ }
+
+ /** replacePathByAnotherTree */
+ public boolean replacePathByAnotherTree(DataTypeAccessTree cast, List<String> path, int index) {
+ if (index >= path.size()) {
+ return true;
+ }
+ if (cast.type instanceof StructType) {
+ List<StructField> fields = ((StructType) cast.type).getFields();
+ for (int i = 0; i < fields.size(); i++) {
+ String castFieldName = path.get(index);
+ if (fields.get(i).getName().equalsIgnoreCase(castFieldName)) {
+ String originFieldName = ((StructType) type).getFields().get(i).getName();
+ path.set(index, originFieldName);
+ return children.get(originFieldName).replacePathByAnotherTree(
+ cast.children.get(castFieldName), path, index + 1
+ );
+ }
+ }
+ } else if (cast.type instanceof ArrayType) {
+ return children.values().iterator().next().replacePathByAnotherTree(
+ cast.children.values().iterator().next(), path, index + 1);
+ } else if (cast.type instanceof MapType) {
+ String fieldName = path.get(index);
+ return children.get("VALUES").replacePathByAnotherTree(
+ cast.children.get(fieldName), path, index + 1
+ );
+ }
+ return false;
+ }
+
+ /** setAccessByPath */
+ public void setAccessByPath(List<String> path, int accessIndex, TAccessPathType pathType) {
+ if (accessIndex >= path.size()) {
+ accessAll = true;
+ return;
+ } else {
+ accessPartialChild = true;
+ }
+
+ if (pathType == TAccessPathType.DATA) {
+ this.pathType = TAccessPathType.DATA;
+ }
+
+ if (this.type.isStructType()) {
+ String fieldName = path.get(accessIndex).toLowerCase();
+ DataTypeAccessTree child = children.get(fieldName);
+ if (child != null) {
+ child.setAccessByPath(path, accessIndex + 1, pathType);
+ } else {
+ // can not find the field
+ accessAll = true;
+ }
+ return;
+ } else if (this.type.isArrayType()) {
+ DataTypeAccessTree child = children.get("*");
+ if (path.get(accessIndex).equals("*")) {
+ // enter this array and skip next *
+ child.setAccessByPath(path, accessIndex + 1, pathType);
+ }
+ return;
+ } else if (this.type.isMapType()) {
+ String fieldName = path.get(accessIndex);
+ if (fieldName.equals("*")) {
+ // access value by the key, so we should access key and access value, then prune the value's type.
+ // e.g. map_column['id'] should access the keys, and access the values
+ DataTypeAccessTree keysChild = children.get("KEYS");
+ DataTypeAccessTree valuesChild = children.get("VALUES");
+ keysChild.accessAll = true;
+ valuesChild.setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ } else if (fieldName.equals("KEYS")) {
+ // only access the keys and not need enter keys, because it must be primitive type.
+ // e.g. map_keys(map_column)
+ DataTypeAccessTree keysChild = children.get("KEYS");
+ keysChild.accessAll = true;
+ return;
+ } else if (fieldName.equals("VALUES")) {
+ // only access the values without keys, and maybe prune the value's data type.
+ // e.g. map_values(map_columns)[0] will access the array of values first,
+ // and then access the array, so the access path is ['VALUES', '*']
+ DataTypeAccessTree valuesChild = children.get("VALUES");
+ valuesChild.setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ }
+ } else if (isRoot) {
+ children.get(path.get(accessIndex).toLowerCase()).setAccessByPath(path, accessIndex + 1, pathType);
+ return;
+ }
+ throw new AnalysisException("unsupported data type: " + this.type);
+ }
+
+ public static DataTypeAccessTree ofRoot(Slot slot, TAccessPathType pathType) {
+ DataTypeAccessTree child = of(slot.getDataType(), pathType);
+ DataTypeAccessTree root = new DataTypeAccessTree(true, NullType.INSTANCE, pathType);
+ root.children.put(slot.getName().toLowerCase(), child);
+ return root;
+ }
+
+ /** of */
+ public static DataTypeAccessTree of(DataType type, TAccessPathType pathType) {
+ DataTypeAccessTree root = new DataTypeAccessTree(type, pathType);
+ if (type instanceof StructType) {
+ StructType structType = (StructType) type;
+ for (Entry<String, StructField> kv : structType.getNameToFields().entrySet()) {
+ root.children.put(kv.getKey().toLowerCase(), of(kv.getValue().getDataType(), pathType));
+ }
+ } else if (type instanceof ArrayType) {
+ root.children.put("*", of(((ArrayType) type).getItemType(), pathType));
+ } else if (type instanceof MapType) {
+ root.children.put("KEYS", of(((MapType) type).getKeyType(), pathType));
+ root.children.put("VALUES", of(((MapType) type).getValueType(), pathType));
+ }
+ return root;
+ }
+
+ /** pruneDataType */
+ public Optional<DataType> pruneDataType() {
+ if (isRoot) {
+ return children.values().iterator().next().pruneDataType();
+ } else if (accessAll) {
+ return Optional.of(type);
+ } else if (!accessPartialChild) {
+ return Optional.empty();
+ }
+
+ List<Pair<String, DataType>> accessedChildren = new ArrayList<>();
+
+ if (type instanceof StructType) {
+ for (Entry<String, DataTypeAccessTree> kv : children.entrySet()) {
+ DataTypeAccessTree childTypeTree = kv.getValue();
+ Optional<DataType> childDataType = childTypeTree.pruneDataType();
+ if (childDataType.isPresent()) {
+ accessedChildren.add(Pair.of(kv.getKey(), childDataType.get()));
+ }
+ }
+ } else if (type instanceof ArrayType) {
+ Optional<DataType> childDataType = children.get("*").pruneDataType();
+ if (childDataType.isPresent()) {
+ accessedChildren.add(Pair.of("*", childDataType.get()));
+ }
+ } else if (type instanceof MapType) {
+ DataType prunedValueType = children.get("VALUES")
+ .pruneDataType()
+ .orElse(((MapType) type).getValueType());
+ // can not prune keys but can prune values
+ accessedChildren.add(Pair.of("KEYS", ((MapType) type).getKeyType()));
+ accessedChildren.add(Pair.of("VALUES", prunedValueType));
+ }
+ if (accessedChildren.isEmpty()) {
+ return Optional.of(type);
+ }
+
+ return Optional.of(pruneDataType(type, accessedChildren));
+ }
+
+ private DataType pruneDataType(DataType dataType, List<Pair<String, DataType>> newChildrenTypes) {
+ if (dataType instanceof StructType) {
+ // prune struct fields
+ StructType structType = (StructType) dataType;
+ Map<String, StructField> nameToFields = structType.getNameToFields();
+ List<StructField> newFields = new ArrayList<>();
+ for (Pair<String, DataType> kv : newChildrenTypes) {
+ String fieldName = kv.first;
+ StructField originField = nameToFields.get(fieldName);
+ DataType prunedType = kv.second;
+ newFields.add(new StructField(
+ originField.getName(), prunedType, originField.isNullable(), originField.getComment()
+ ));
+ }
+ return new StructType(newFields);
+ } else if (dataType instanceof ArrayType) {
+ return ArrayType.of(newChildrenTypes.get(0).second, ((ArrayType) dataType).containsNull());
+ } else if (dataType instanceof MapType) {
+ return MapType.of(newChildrenTypes.get(0).second, newChildrenTypes.get(1).second);
+ } else {
+ throw new AnalysisException("unsupported data type: " + dataType);
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
new file mode 100644
index 00000000000..6ef42183c7b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
@@ -0,0 +1,315 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.pattern.MatchingContext;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+
+/** PushDownNestedColumnThroughJoin */
+public class PushDownNestedColumn implements RewriteRuleFactory {
+ @Override
+ public List<Rule> buildRules() {
+ return ImmutableList.of(
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN.build(
+ logicalProject(logicalJoin()).thenApply(this::defaultPushDown)
+ ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_WINDOW.build(
+ logicalProject(logicalWindow()).thenApply(this::defaultPushDown)
+ ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_PARTITION_TOP_N.build(
+ logicalProject(logicalPartitionTopN()).thenApply(this::defaultPushDown)
+ ),
+ // RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_DEFER_MATERIALIZE_TOP_N.build(
+ // logicalProject(logicalDeferMaterializeTopN()).thenApply(this::defaultPushDown)
+ // ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION.build(
+ logicalProject(
+ logicalUnion().when(u -> u.getQualifier() == Qualifier.ALL)
+ ).thenApply(this::pushThroughUnion)
+ )
+ );
+ }
+
+ private <C extends LogicalPlan> Plan defaultPushDown(MatchingContext<LogicalProject<C>> ctx) {
+ if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) {
+ return ctx.root;
+ }
+
+ LogicalProject<C> project = ctx.root;
+ C child = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, child);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ = pushdownProjectHelper.pushDownExpressions(project.getProjects());
+
+ if (pushProjects.first) {
+ List<Plan> newJoinChildren = pushdownProjectHelper.buildNewChildren();
+ return new LogicalProject<>(
+ pushProjects.second,
+ child.withChildren(newJoinChildren)
+ );
+ }
+ return project;
+ }
+
+ private Plan pushThroughUnion(MatchingContext<LogicalProject<LogicalUnion>> ctx) {
+ if (!ctx.connectContext.getSessionVariable().enablePruneNestedColumns) {
+ return ctx.root;
+ }
+ LogicalProject<LogicalUnion> project = ctx.root;
+ LogicalUnion union = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, project);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ = pushdownProjectHelper.pushDownExpressions(project.getProjects());
+ if (pushProjects.first) {
+ List<NamedExpression> unionOutputs = union.getOutputs();
+ Map<Slot, Integer> slotToColumnIndex = new LinkedHashMap<>();
+ for (int i = 0; i < unionOutputs.size(); i++) {
+ NamedExpression output = unionOutputs.get(i);
+ slotToColumnIndex.put(output.toSlot(), i);
+ }
+
+ Collection<NamedExpression> pushDownProjections
+ = pushdownProjectHelper.childToPushDownProjects.values();
+ List<Plan> newChildren = new ArrayList<>();
+ List<List<SlotReference>> newChildrenOutputs = new ArrayList<>();
+ for (Plan child : union.children()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex = slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return child.getOutput().get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ child
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ for (List<NamedExpression> originConstantExprs : union.getConstantExprsList()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex = slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return originConstantExprs.get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalOneRowRelation originOneRowRelation = new LogicalOneRowRelation(
+ ctx.statementContext.getNextRelationId(),
+ originConstantExprs
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(originOneRowRelation.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ originOneRowRelation
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ List<NamedExpression> newUnionOutputs = new ArrayList<>(union.getOutputs());
+ for (NamedExpression projection : pushDownProjections) {
+ newUnionOutputs.add(projection.toSlot());
+ }
+
+ return new LogicalProject<>(
+ pushProjects.second,
+ new LogicalUnion(
+ union.getQualifier(),
+ newUnionOutputs,
+ newChildrenOutputs,
+ ImmutableList.of(),
+ union.hasPushedFilter(),
+ newChildren
+ )
+ );
+ }
+ return project;
+ }
+
+ private List<NamedExpression> replaceSlot(
+ StatementContext statementContext,
+ Collection<NamedExpression> pushDownProjections,
+ Function<Slot, Slot> slotReplace) {
+ List<NamedExpression> pushedOutput = new ArrayList<>();
+ for (NamedExpression projection : pushDownProjections) {
+ NamedExpression newOutput = (NamedExpression) projection.rewriteUp(e -> {
+ if (e instanceof Slot) {
+ Slot newSlot = slotReplace.apply((Slot) e);
+ if (newSlot != null) {
+ return newSlot;
+ }
+ }
+ return e;
+ });
+ if (newOutput instanceof Alias) {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(), newOutput.child(0)));
+ } else {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(), newOutput));
+ }
+ }
+ return pushedOutput;
+ }
+
+ private static class PushdownProjectHelper {
+ private final Plan plan;
+ private final StatementContext statementContext;
+ private final Map<Expression, Pair<Slot, Plan>> exprToChildAndSlot;
+ private final Multimap<Plan, NamedExpression> childToPushDownProjects;
+
+ public PushdownProjectHelper(StatementContext statementContext, Plan plan) {
+ this.statementContext = statementContext;
+ this.plan = plan;
+ this.exprToChildAndSlot = new LinkedHashMap<>();
+ this.childToPushDownProjects = ArrayListMultimap.create();
+ }
+
+ public <C extends Collection<E>, E extends Expression> Pair<Boolean, C> pushDownExpressions(C expressions) {
+ ImmutableCollection.Builder<E> builder;
+ if (expressions instanceof List) {
+ builder = ImmutableList.builderWithExpectedSize(expressions.size());
+ } else {
+ builder = ImmutableSet.builderWithExpectedSize(expressions.size());
+ }
+
+ boolean extracted = false;
+ for (E expression : expressions) {
+ Optional<E> result = pushDownExpression(expression);
+ if (!result.isPresent()) {
+ builder.add(expression);
+ } else {
+ extracted = true;
+ builder.add(result.get());
+ }
+ }
+
+ if (extracted) {
+ return Pair.of(true, (C) builder.build());
+ } else {
+ return Pair.of(false, expressions);
+ }
+ }
+
+ public <E extends Expression> Optional<E> pushDownExpression(E expression) {
+ if (!(expression instanceof PreferPushDownProject
+ || (expression instanceof Alias && expression.child(0) instanceof PreferPushDownProject))) {
+ return Optional.empty();
+ }
+ Pair<Slot, Plan> existPushdown = exprToChildAndSlot.get(expression);
+ if (existPushdown != null) {
+ return Optional.of((E) existPushdown.first);
+ }
+
+ Alias pushDownAlias = null;
+ if (expression instanceof Alias) {
+ pushDownAlias = (Alias) expression;
+ } else {
+ pushDownAlias = new Alias(statementContext.getNextExprId(), expression);
+ }
+
+ Set<Slot> inputSlots = expression.getInputSlots();
+ for (Plan child : plan.children()) {
+ if (child.getOutputSet().containsAll(inputSlots)) {
+ Slot remaimSlot = pushDownAlias.toSlot();
+ exprToChildAndSlot.put(expression, Pair.of(remaimSlot, child));
+ childToPushDownProjects.put(child, pushDownAlias);
+ return Optional.of((E) remaimSlot);
+ }
+ }
+ return Optional.empty();
+ }
+
+ public List<Plan> buildNewChildren() {
+ if (childToPushDownProjects.isEmpty()) {
+ return plan.children();
+ }
+ ImmutableList.Builder<Plan> newChildren = ImmutableList.builderWithExpectedSize(plan.arity());
+ for (Plan child : plan.children()) {
+ Collection<NamedExpression> newProject = childToPushDownProjects.get(child);
+ if (newProject.isEmpty()) {
+ newChildren.add(child);
+ } else {
+ newChildren.add(
+ new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(newProject)
+ .build(),
+ child
+ )
+ );
+ }
+ }
+ return newChildren.build();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
new file mode 100644
index 00000000000..459c7362758
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -0,0 +1,674 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.AccessPathInfo;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
+import org.apache.doris.nereids.properties.OrderKey;
+import org.apache.doris.nereids.rules.rewrite.NestedColumnPruning.DataTypeAccessTree;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Cast;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.OrderExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
+import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
+import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
+import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.MapType;
+import org.apache.doris.nereids.types.NestedColumnPrunable;
+import org.apache.doris.nereids.types.StructType;
+import org.apache.doris.nereids.util.MoreFieldsThread;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.ImmutableMultimap.Builder;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+
+/** SlotTypeReplacer */
+public class SlotTypeReplacer extends DefaultPlanRewriter<Void> {
+ private Map<Integer, AccessPathInfo> replacedDataTypes;
+
+ public SlotTypeReplacer(Map<Integer, AccessPathInfo> bottomReplacedDataTypes) {
+ this.replacedDataTypes = Maps.newLinkedHashMap(bottomReplacedDataTypes);
+ }
+
+ @Override
+ public Plan visitLogicalCTEProducer(LogicalCTEProducer<? extends Plan> cteProducer, Void context) {
+ return super.visitLogicalCTEProducer(cteProducer, context);
+ }
+
+ @Override
+ public Plan visitLogicalWindow(LogicalWindow<? extends Plan> window, Void context) {
+ window = visitChildren(this, window, context);
+ Pair<Boolean, ? extends List<? extends Expression>> replaced = replaceExpressions(
+ window.getExpressions(), false, false);
+ if (replaced.first) {
+ return window.withExpressionsAndChild((List) replaced.second, window.child());
+ }
+ return window;
+ }
+
+ @Override
+ public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, Void context) {
+ Map<Slot, Slot> consumerToProducerOutputMap = cteConsumer.getConsumerToProducerOutputMap();
+ Multimap<Slot, Slot> producerToConsumerOutputMap = cteConsumer.getProducerToConsumerOutputMap();
+
+ Map<Slot, Slot> replacedConsumerToProducerOutputMap = new LinkedHashMap<>();
+ Builder<Slot, Slot> replacedProducerToConsumerOutputMap = ImmutableMultimap.builder();
+
+ boolean changed = false;
+ for (Entry<Slot, Slot> kv : consumerToProducerOutputMap.entrySet()) {
+ Slot consumerSlot = kv.getKey();
+ Slot producerSlot = kv.getValue();
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(producerSlot.getExprId().asInt());
+ if (accessPathInfo != null) {
+ DataType prunedType = accessPathInfo.getPrunedType();
+ if (!prunedType.equals(producerSlot.getDataType())) {
+ replacedDataTypes.put(consumerSlot.getExprId().asInt(), accessPathInfo);
+ changed = true;
+ producerSlot = producerSlot.withNullableAndDataType(producerSlot.nullable(), prunedType);
+ consumerSlot = consumerSlot.withNullableAndDataType(consumerSlot.nullable(), prunedType);
+ }
+ }
+ replacedConsumerToProducerOutputMap.put(consumerSlot, producerSlot);
+ }
+
+ for (Entry<Slot, Collection<Slot>> kv : producerToConsumerOutputMap.asMap().entrySet()) {
+ Slot producerSlot = kv.getKey();
+ Collection<Slot> consumerSlots = kv.getValue();
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(producerSlot.getExprId().asInt());
+ if (accessPathInfo != null && !accessPathInfo.getPrunedType().equals(producerSlot.getDataType())) {
+ DataType replacedDataType = accessPathInfo.getPrunedType();
+ changed = true;
+ producerSlot = producerSlot.withNullableAndDataType(producerSlot.nullable(), replacedDataType);
+ for (Slot consumerSlot : consumerSlots) {
+ consumerSlot = consumerSlot.withNullableAndDataType(consumerSlot.nullable(), replacedDataType);
+ replacedProducerToConsumerOutputMap.put(producerSlot, consumerSlot);
+ }
+ } else {
+ replacedProducerToConsumerOutputMap.putAll(producerSlot, consumerSlots);
+ }
+ }
+
+ if (changed) {
+ return new LogicalCTEConsumer(
+ cteConsumer.getRelationId(), cteConsumer.getCteId(), cteConsumer.getName(),
+ replacedConsumerToProducerOutputMap, replacedProducerToConsumerOutputMap.build()
+ );
+ }
+ return cteConsumer;
+ }
+
+ @Override
+ public Plan visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join, Void context) {
+ join = visitChildren(this, join, context);
+ Pair<Boolean, List<Expression>> replacedHashJoinConjuncts
+ = replaceExpressions(join.getHashJoinConjuncts(), false, false);
+ Pair<Boolean, List<Expression>> replacedOtherJoinConjuncts
+ = replaceExpressions(join.getOtherJoinConjuncts(), false, false);
+
+ if (replacedHashJoinConjuncts.first || replacedOtherJoinConjuncts.first) {
+ return join.withJoinConjuncts(
+ replacedHashJoinConjuncts.second,
+ replacedOtherJoinConjuncts.second,
+ join.getJoinReorderContext());
+ }
+ return join;
+ }
+
+ @Override
+ public Plan visitLogicalProject(LogicalProject<? extends Plan> project, Void context) {
+ project = visitChildren(this, project, context);
+
+ Pair<Boolean, List<NamedExpression>> projects = replaceExpressions(project.getProjects(), true, false);
+ if (projects.first) {
+ return project.withProjects(projects.second);
+ }
+ return project;
+ }
+
+ @Override
+ public Plan visitLogicalPartitionTopN(LogicalPartitionTopN<? extends Plan> partitionTopN, Void context) {
+ partitionTopN = visitChildren(this, partitionTopN, context);
+
+ Pair<Boolean, List<Expression>> replacedPartitionKeys = replaceExpressions(
+ partitionTopN.getPartitionKeys(), false, false);
+ Pair<Boolean, List<OrderExpression>> replacedOrderExpressions
+ = replaceOrderExpressions(partitionTopN.getOrderKeys());
+ if (replacedPartitionKeys.first || replacedOrderExpressions.first) {
+ return partitionTopN.withPartitionKeysAndOrderKeys(
+ replacedPartitionKeys.second, replacedOrderExpressions.second);
+ }
+ return partitionTopN;
+ }
+
+ @Override
+ public Plan visitLogicalDeferMaterializeTopN(LogicalDeferMaterializeTopN<? extends Plan> topN, Void context) {
+ topN = visitChildren(this, topN, context);
+
+ LogicalTopN logicalTopN = (LogicalTopN) topN.getLogicalTopN().accept(this, context);
+ if (logicalTopN != topN.getLogicalTopN()) {
+ SlotReference replacedColumnIdSlot = replaceExpressions(
+ ImmutableList.of(topN.getColumnIdSlot()), false, false).second.get(0);
+ return new LogicalDeferMaterializeTopN(
+ logicalTopN, topN.getDeferMaterializeSlotIds(), replacedColumnIdSlot);
+ }
+
+ return topN;
+ }
+
+ @Override
+ public Plan visitLogicalExcept(LogicalExcept except, Void context) {
+ except = visitChildren(this, except, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ except.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(except.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalExcept(except.getQualifier(), except.getOutputs(),
+ except.getRegularChildrenOutputs(), except.children());
+ }
+
+ return except;
+ }
+
+ @Override
+ public Plan visitLogicalIntersect(LogicalIntersect intersect, Void context) {
+ intersect = visitChildren(this, intersect, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ intersect.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(intersect.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalIntersect(intersect.getQualifier(), intersect.getOutputs(),
+ intersect.getRegularChildrenOutputs(), intersect.children());
+ }
+ return intersect;
+ }
+
+ @Override
+ public Plan visitLogicalUnion(LogicalUnion union, Void context) {
+ union = visitChildren(this, union, context);
+
+ Pair<Boolean, List<List<SlotReference>>> replacedRegularChildrenOutputs = replaceMultiExpressions(
+ union.getRegularChildrenOutputs());
+
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(union.getOutputs(), true, false);
+
+ if (replacedRegularChildrenOutputs.first || replacedOutputs.first) {
+ return new LogicalUnion(
+ union.getQualifier(),
+ replacedOutputs.second,
+ replacedRegularChildrenOutputs.second,
+ union.getConstantExprsList(),
+ union.hasPushedFilter(),
+ union.children()
+ );
+ }
+
+ return union;
+ }
+
+ @Override
+ public Plan visitLogicalRepeat(LogicalRepeat<? extends Plan> repeat, Void context) {
+ repeat = visitChildren(this, repeat, context);
+
+ Pair<Boolean, List<List<Expression>>> replacedGroupingSets
+ = replaceMultiExpressions(repeat.getGroupingSets());
+ Pair<Boolean, List<NamedExpression>> replacedOutputs
+ = replaceExpressions(repeat.getOutputExpressions(), true, false);
+
+ if (replacedGroupingSets.first || replacedOutputs.first) {
+ return repeat.withGroupSetsAndOutput(replacedGroupingSets.second, replacedOutputs.second);
+ }
+ return repeat;
+ }
+
+ @Override
+ public Plan visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, Void context) {
+ generate = visitChildren(this, generate, context);
+
+ Pair<Boolean, List<Function>> replacedGenerators
+ = replaceExpressions(generate.getGenerators(), false, false);
+ Pair<Boolean, List<Slot>> replacedGeneratorOutput
+ = replaceExpressions(generate.getGeneratorOutput(), false, false);
+ if (replacedGenerators.first || replacedGeneratorOutput.first) {
+ return new LogicalGenerate<>(replacedGenerators.second, replacedGeneratorOutput.second,
+ generate.getExpandColumnAlias(), generate.child());
+ }
+ return generate;
+ }
+
+ @Override
+ public Plan visitLogicalAggregate(LogicalAggregate<? extends Plan> aggregate, Void context) {
+ aggregate = visitChildren(this, aggregate, context);
+
+ Pair<Boolean, List<Expression>> replacedGroupBy = replaceExpressions(
+ aggregate.getGroupByExpressions(), false, false);
+ Pair<Boolean, List<NamedExpression>> replacedOutput = replaceExpressions(
+ aggregate.getOutputExpressions(), true, false);
+
+ if (replacedGroupBy.first || replacedOutput.first) {
+ return aggregate.withGroupByAndOutput(replacedGroupBy.second, replacedOutput.second);
+ }
+ return aggregate;
+ }
+
+ @Override
+ public Plan visitLogicalSort(LogicalSort<? extends Plan> sort, Void context) {
+ sort = visitChildren(this, sort, context);
+
+ Pair<Boolean, List<OrderKey>> replaced = replaceOrderKeys(sort.getOrderKeys());
+ if (replaced.first) {
+ return sort.withOrderKeys(replaced.second);
+ }
+ return sort;
+ }
+
+ @Override
+ public Plan visitLogicalTopN(LogicalTopN<? extends Plan> topN, Void context) {
+ topN = visitChildren(this, topN, context);
+
+ Pair<Boolean, List<OrderKey>> replaced = replaceOrderKeys(topN.getOrderKeys());
+ if (replaced.first) {
+ return topN.withOrderKeys(replaced.second);
+ }
+ return topN;
+ }
+
+ @Override
+ public Plan visitLogicalDeferMaterializeOlapScan(
+ LogicalDeferMaterializeOlapScan deferMaterializeOlapScan, Void context) {
+
+ LogicalOlapScan logicalOlapScan
+ = (LogicalOlapScan) deferMaterializeOlapScan.getLogicalOlapScan().accept(this, context);
+
+ if (logicalOlapScan != deferMaterializeOlapScan.getLogicalOlapScan()) {
+ SlotReference replacedColumnIdSlot = replaceExpressions(
+ ImmutableList.of(deferMaterializeOlapScan.getColumnIdSlot()), false, false).second.get(0);
+ return new LogicalDeferMaterializeOlapScan(
+ logicalOlapScan, deferMaterializeOlapScan.getDeferMaterializeSlotIds(), replacedColumnIdSlot
+ );
+ }
+ return deferMaterializeOlapScan;
+ }
+
+ @Override
+ public Plan visitLogicalFilter(LogicalFilter<? extends Plan> filter, Void context) {
+ filter = visitChildren(this, filter, context);
+
+ Pair<Boolean, Set<Expression>> replaced = replaceExpressions(filter.getConjuncts(), false, false);
+ if (replaced.first) {
+ return filter.withConjuncts(replaced.second);
+ }
+ return filter;
+ }
+
+ @Override
+ public Plan visitLogicalFileScan(LogicalFileScan fileScan, Void context) {
+ Pair<Boolean, List<Slot>> replaced = replaceExpressions(fileScan.getOutput(), false, true);
+ if (replaced.first) {
+ List<Slot> replaceSlots = new ArrayList<>(replaced.second);
+ if (fileScan.getTable() instanceof IcebergExternalTable) {
+ for (int i = 0; i < replaceSlots.size(); i++) {
+ Slot slot = replaceSlots.get(i);
+ if (!(slot instanceof SlotReference)) {
+ continue;
+ }
+ SlotReference slotReference = (SlotReference) slot;
+ Optional<List<TColumnAccessPath>> allAccessPaths = slotReference.getAllAccessPaths();
+ if (!allAccessPaths.isPresent() || !slotReference.getOriginalColumn().isPresent()) {
+ continue;
+ }
+ List<TColumnAccessPath> allAccessPathsWithId
+ = replaceIcebergAccessPathToId(allAccessPaths.get(), slotReference);
+ List<TColumnAccessPath> predicateAccessPathsWithId = replaceIcebergAccessPathToId(
+ slotReference.getPredicateAccessPaths().get(), slotReference);
+ replaceSlots.set(i, ((SlotReference) slot).withAccessPaths(
+ allAccessPathsWithId,
+ predicateAccessPathsWithId,
+ allAccessPaths.get(),
+ slotReference.getPredicateAccessPaths().get()
+ ));
+ }
+ }
+ return fileScan.withCachedOutput(replaceSlots);
+ }
+ return fileScan;
+ }
+
+ @Override
+ public Plan visitLogicalTVFRelation(LogicalTVFRelation tvfRelation, Void context) {
+ Pair<Boolean, List<Slot>> replaced
+ = replaceExpressions(tvfRelation.getOutput(), false, true);
+ if (replaced.first) {
+ return tvfRelation.withCachedOutputs(replaced.second);
+ }
+ return tvfRelation;
+ }
+
+ @Override
+ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Void context) {
+ Pair<Boolean, List<Slot>> replaced = replaceExpressions(olapScan.getOutput(), false, true);
+ if (replaced.first) {
+ return olapScan.withPrunedTypeSlots(replaced.second);
+ }
+ return olapScan;
+ }
+
+ @Override
+ public Plan visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, Void context) {
+ Pair<Boolean, List<NamedExpression>> replacedProjects
+ = replaceExpressions(emptyRelation.getProjects(), true, false);
+
+ if (replacedProjects.first) {
+ return emptyRelation.withProjects(replacedProjects.second);
+ }
+ return emptyRelation;
+ }
+
+ @Override
+ public Plan visitLogicalOneRowRelation(LogicalOneRowRelation oneRowRelation, Void context) {
+ Pair<Boolean, List<NamedExpression>> replacedProjects
+ = replaceExpressions(oneRowRelation.getProjects(), true, false);
+
+ if (replacedProjects.first) {
+ return oneRowRelation.withProjects(replacedProjects.second);
+ }
+ return oneRowRelation;
+ }
+
+ @Override
+ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResultSink, Void context) {
+ logicalResultSink = visitChildren(this, logicalResultSink, context);
+
+ Pair<Boolean, List<NamedExpression>> replacedOutput = replaceExpressions(logicalResultSink.getOutputExprs(),
+ false, false);
+ if (replacedOutput.first) {
+ return logicalResultSink.withOutputExprs(replacedOutput.second);
+ }
+ return logicalResultSink;
+ }
+
+ private Pair<Boolean, List<OrderExpression>> replaceOrderExpressions(List<OrderExpression> orderExpressions) {
+ ImmutableList.Builder<OrderExpression> newOrderKeys
+ = ImmutableList.builderWithExpectedSize(orderExpressions.size());
+ boolean changed = false;
+ for (OrderExpression orderExpression : orderExpressions) {
+ Expression newOrderKeyExpr = replaceSlot(orderExpression.getOrderKey().getExpr(), false);
+ if (newOrderKeyExpr != orderExpression.getOrderKey().getExpr()) {
+ newOrderKeys.add(new OrderExpression(orderExpression.getOrderKey().withExpression(newOrderKeyExpr)));
+ changed = true;
+ } else {
+ newOrderKeys.add(orderExpression);
+ }
+ }
+ return Pair.of(changed, newOrderKeys.build());
+ }
+
+ private Pair<Boolean, List<OrderKey>> replaceOrderKeys(List<OrderKey> orderKeys) {
+ ImmutableList.Builder<OrderKey> newOrderKeys = ImmutableList.builderWithExpectedSize(orderKeys.size());
+ boolean changed = false;
+ for (OrderKey orderKey : orderKeys) {
+ Expression newOrderKeyExpr = replaceSlot(orderKey.getExpr(), false);
+ if (newOrderKeyExpr != orderKey.getExpr()) {
+ newOrderKeys.add(orderKey.withExpression(newOrderKeyExpr));
+ changed = true;
+ } else {
+ newOrderKeys.add(orderKey);
+ }
+ }
+ return Pair.of(changed, newOrderKeys.build());
+ }
+
+ private <C extends Collection<E>, E extends Expression>
+ Pair<Boolean, List<C>> replaceMultiExpressions(List<C> expressionsList) {
+ ImmutableList.Builder<C> result = ImmutableList.builderWithExpectedSize(expressionsList.size());
+ boolean changed = false;
+ for (C expressions : expressionsList) {
+ Pair<Boolean, C> replaced = replaceExpressions(expressions, false, false);
+ changed |= replaced.first;
+ result.add(replaced.second);
+ }
+ return Pair.of(changed, result.build());
+ }
+
+ private <C extends Collection<E>, E extends Expression> Pair<Boolean, C> replaceExpressions(
+ C expressions, boolean propagateType, boolean fillAccessPaths) {
+ ImmutableCollection.Builder<E> newExprs;
+ if (expressions instanceof List) {
+ newExprs = ImmutableList.builder();
+ } else {
+ newExprs = ImmutableSet.builder();
+ }
+
+ boolean changed = false;
+ for (Expression oldExpr : expressions) {
+ Expression newExpr = replaceSlot(oldExpr, fillAccessPaths);
+ if (newExpr != oldExpr) {
+ newExprs.add((E) newExpr);
+ changed = true;
+
+ if (propagateType && oldExpr instanceof NamedExpression
+ && !oldExpr.getDataType().equals(newExpr.getDataType())) {
+ replacedDataTypes.put(
+ ((NamedExpression) oldExpr).getExprId().asInt(),
+ // not need access path in the upper slots
+ new AccessPathInfo(newExpr.getDataType(), null, null)
+ );
+ }
+ } else {
+ newExprs.add((E) oldExpr);
+ }
+ }
+ return Pair.of(changed, (C) newExprs.build());
+ }
+
+ private Expression replaceSlot(Expression e, boolean fillAccessPath) {
+ return MoreFieldsThread.keepFunctionSignature(false,
+ () -> doRewriteExpression(e, fillAccessPath)
+ );
+ }
+
+ private Expression doRewriteExpression(Expression e, boolean fillAccessPath) {
+ if (e instanceof Lambda) {
+ return rewriteLambda((Lambda) e, fillAccessPath);
+ } else if (e instanceof Cast) {
+ return rewriteCast((Cast) e, fillAccessPath);
+ } else if (e instanceof SlotReference) {
+ AccessPathInfo accessPathInfo = replacedDataTypes.get(((SlotReference) e).getExprId().asInt());
+ if (accessPathInfo != null) {
+ SlotReference newSlot = (SlotReference) ((SlotReference) e).withNullableAndDataType(
+ e.nullable(), accessPathInfo.getPrunedType());
+ if (fillAccessPath) {
+ newSlot = newSlot.withAccessPaths(
+ accessPathInfo.getAllAccessPaths(), accessPathInfo.getPredicateAccessPaths()
+ );
+ }
+ return newSlot;
+ }
+ }
+
+ ImmutableList.Builder<Expression> newChildren = ImmutableList.builderWithExpectedSize(e.arity());
+ boolean changed = false;
+ for (Expression child : e.children()) {
+ Expression newChild = doRewriteExpression(child, fillAccessPath);
+ changed |= child != newChild;
+ newChildren.add(newChild);
+ }
+ return changed ? e.withChildren(newChildren.build()) : e;
+ }
+
+ private Expression rewriteLambda(Lambda e, boolean fillAccessPath) {
+ // we should rewrite ArrayItemReference first, then we can replace the ArrayItemSlot int the lambda
+ Expression[] newChildren = new Expression[e.arity()];
+ for (int i = 0; i < e.arity(); i++) {
+ Expression child = e.child(i);
+ if (child instanceof ArrayItemReference) {
+ Expression newRef = child.withChildren(doRewriteExpression(child.child(0), fillAccessPath));
+ replacedDataTypes.put(((ArrayItemReference) child).getExprId().asInt(),
+ new AccessPathInfo(newRef.getDataType(), null, null));
+ newChildren[i] = newRef;
+ } else {
+ newChildren[i] = child;
+ }
+ }
+
+ for (int i = 0; i < newChildren.length; i++) {
+ Expression child = newChildren[i];
+ if (!(child instanceof ArrayItemReference)) {
+ newChildren[i] = doRewriteExpression(child, fillAccessPath);
+ }
+ }
+
+ return e.withChildren(newChildren);
+ }
+
+ private Expression rewriteCast(Cast cast, boolean fillAccessPath) {
+ Expression newChild = doRewriteExpression(cast.child(0), fillAccessPath);
+ if (newChild == cast.child(0)) {
+ return cast;
+ }
+
+ DataType newType = cast.getDataType();
+ if (cast.getDataType() instanceof NestedColumnPrunable
+ && newChild.getDataType() instanceof NestedColumnPrunable) {
+ DataTypeAccessTree originTree = DataTypeAccessTree.of(cast.child().getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree prunedTree = DataTypeAccessTree.of(newChild.getDataType(), TAccessPathType.DATA);
+ DataTypeAccessTree castTree = DataTypeAccessTree.of(cast.getDataType(), TAccessPathType.DATA);
+ newType = prunedTree.pruneCastType(originTree, castTree);
+ }
+
+ return new Cast(newChild, newType);
+ }
+
+ private List<TColumnAccessPath> replaceIcebergAccessPathToId(
+ List<TColumnAccessPath> originAccessPaths, SlotReference slotReference) {
+ Column column = slotReference.getOriginalColumn().get();
+ List<TColumnAccessPath> replacedAccessPaths = new ArrayList<>();
+ for (TColumnAccessPath accessPath : originAccessPaths) {
+ List<String> icebergColumnAccessPath = new ArrayList<>();
+ if (accessPath.type == TAccessPathType.DATA) {
+ icebergColumnAccessPath.addAll(accessPath.data_access_path.path);
+ replaceIcebergAccessPathToId(
+ icebergColumnAccessPath, 0, slotReference.getDataType(), column
+ );
+ TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ newAccessPath.data_access_path = new TDataAccessPath(icebergColumnAccessPath);
+ replacedAccessPaths.add(newAccessPath);
+ } else {
+ icebergColumnAccessPath.addAll(accessPath.meta_access_path.path);
+ replaceIcebergAccessPathToId(
+ icebergColumnAccessPath, 0, slotReference.getDataType(), column
+ );
+ TColumnAccessPath newAccessPath = new TColumnAccessPath(TAccessPathType.META);
+ newAccessPath.meta_access_path = new TMetaAccessPath(icebergColumnAccessPath);
+ replacedAccessPaths.add(newAccessPath);
+ }
+ }
+ return replacedAccessPaths;
+ }
+
+ private void replaceIcebergAccessPathToId(List<String> originPath, int index, DataType type, Column column) {
+ if (index >= originPath.size()) {
+ return;
+ }
+ if (index == 0) {
+ originPath.set(index, String.valueOf(column.getUniqueId()));
+ replaceIcebergAccessPathToId(originPath, index + 1, type, column);
+ } else {
+ String fieldName = originPath.get(index);
+ if (type instanceof ArrayType) {
+ // skip replace *
+ replaceIcebergAccessPathToId(
+ originPath, index + 1, ((ArrayType) type).getItemType(), column.getChildren().get(0)
+ );
+ } else if (type instanceof MapType) {
+ if (fieldName.equals("*") || fieldName.equals("VALUES")) {
+ replaceIcebergAccessPathToId(
+ originPath, index + 1, ((MapType) type).getValueType(), column.getChildren().get(1)
+ );
+ }
+ } else if (type instanceof StructType) {
+ for (Column child : column.getChildren()) {
+ if (child.getName().equals(fieldName)) {
+ originPath.set(index, String.valueOf(child.getUniqueId()));
+ DataType childType = ((StructType) type).getNameToFields().get(fieldName).getDataType();
+ replaceIcebergAccessPathToId(originPath, index + 1, childType, child);
+ break;
+ }
+ }
+ } else {
+ originPath.set(index, String.valueOf(column.getUniqueId()));
+ }
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
index 1ed8d151270..abb65a283ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
@@ -22,7 +22,6 @@ import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.properties.OrderKey;
-import org.apache.doris.nereids.rules.rewrite.ColumnPruning.PruneContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -87,7 +86,7 @@ import java.util.Set;
* generating the slots for the required sub path on scan, union, and cte consumer.
* Then, it replaces the element_at with the corresponding slot.
*/
-public class VariantSubPathPruning extends DefaultPlanRewriter<PruneContext> implements CustomRewriter {
+public class VariantSubPathPruning implements CustomRewriter {
public static final Logger LOG = LogManager.getLogger(VariantSubPathPruning.class);
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
new file mode 100644
index 00000000000..4c9c3bebe5a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+
+/**
+ * some expressions prefer push down project under the plan,
+ *
+ * e.g. the element_at function prefer push down to a project in under the plan:
+ * Project(element_at(left.column, 'id') as a)
+ * |
+ * Join
+ * / |
+ * left right
+ *
+ * we will push down project through the Join, so we can prune the column which has complex type:
+ *
+ * Join
+ * / \
+ * Project(element_at(left.column, 'id') as a, ...) right
+ * |
+ * left
+ */
+public interface PreferPushDownProject extends ExpressionTrait {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
index ce2d3dbb3e5..1c77dc669ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.exceptions.UnboundException;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.thrift.TColumnAccessPath;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -56,6 +57,10 @@ public class SlotReference extends Slot {
// that need return original table and name for view not its original table if u query a view
private final TableIf oneLevelTable;
private final Column oneLevelColumn;
+ private final Optional<List<TColumnAccessPath>> allAccessPaths;
+ private final Optional<List<TColumnAccessPath>> predicateAccessPaths;
+ private final Optional<List<TColumnAccessPath>> displayAllAccessPaths;
+ private final Optional<List<TColumnAccessPath>> displayPredicateAccessPaths;
public SlotReference(String name, DataType dataType) {
this(StatementScopeIdGenerator.newExprId(), name, dataType, true, ImmutableList.of(),
@@ -92,6 +97,15 @@ public class SlotReference extends Slot {
subPath, Optional.empty());
}
+ public SlotReference(ExprId exprId, Supplier<String> name, DataType dataType, boolean nullable,
+ List<String> qualifier, @Nullable TableIf originalTable, @Nullable Column originalColumn,
+ @Nullable TableIf oneLevelTable, Column oneLevelColumn,
+ List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql) {
+ this(exprId, name, dataType, nullable, qualifier, originalTable, originalColumn, oneLevelTable,
+ oneLevelColumn, subPath, indexInSql, Optional.empty(), Optional.empty(),
+ Optional.empty(), Optional.empty());
+ }
+
/**
* Constructor for SlotReference.
*
@@ -106,7 +120,10 @@ public class SlotReference extends Slot {
public SlotReference(ExprId exprId, Supplier<String> name, DataType dataType, boolean nullable,
List<String> qualifier, @Nullable TableIf originalTable, @Nullable Column originalColumn,
@Nullable TableIf oneLevelTable, Column oneLevelColumn,
- List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql) {
+ List<String> subPath, Optional<Pair<Integer, Integer>> indexInSql,
+ Optional<List<TColumnAccessPath>> allAccessPaths, Optional<List<TColumnAccessPath>> predicateAccessPaths,
+ Optional<List<TColumnAccessPath>> displayAllAccessPaths,
+ Optional<List<TColumnAccessPath>> displayPredicateAccessPaths) {
super(indexInSql);
this.exprId = exprId;
this.name = name;
@@ -119,6 +136,10 @@ public class SlotReference extends Slot {
this.oneLevelTable = oneLevelTable;
this.oneLevelColumn = oneLevelColumn;
this.subPath = Objects.requireNonNull(subPath, "subPath can not be null");
+ this.allAccessPaths = allAccessPaths;
+ this.predicateAccessPaths = predicateAccessPaths;
+ this.displayAllAccessPaths = displayAllAccessPaths;
+ this.displayPredicateAccessPaths = displayPredicateAccessPaths;
}
public static SlotReference of(String name, DataType type) {
@@ -342,4 +363,37 @@ public class SlotReference extends Slot {
public boolean hasAutoInc() {
return originalColumn != null ? originalColumn.isAutoInc() : false;
}
+
+ public SlotReference withAccessPaths(
+ List<TColumnAccessPath> allAccessPaths, List<TColumnAccessPath> predicateAccessPaths) {
+ return new SlotReference(exprId, name, dataType, nullable, qualifier,
+ originalTable, originalColumn, oneLevelTable, oneLevelColumn,
+ subPath, indexInSqlString, Optional.of(allAccessPaths), Optional.of(predicateAccessPaths),
+ Optional.of(allAccessPaths), Optional.of(predicateAccessPaths));
+ }
+
+ public SlotReference withAccessPaths(
+ List<TColumnAccessPath> allAccessPaths, List<TColumnAccessPath> predicateAccessPaths,
+ List<TColumnAccessPath> displayAllAccessPaths, List<TColumnAccessPath> displayPredicateAccessPaths) {
+ return new SlotReference(exprId, name, dataType, nullable, qualifier,
+ originalTable, originalColumn, oneLevelTable, oneLevelColumn,
+ subPath, indexInSqlString, Optional.of(allAccessPaths), Optional.of(predicateAccessPaths),
+ Optional.of(displayAllAccessPaths), Optional.of(displayPredicateAccessPaths));
+ }
+
+ public Optional<List<TColumnAccessPath>> getAllAccessPaths() {
+ return allAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getPredicateAccessPaths() {
+ return predicateAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getDisplayAllAccessPaths() {
+ return displayAllAccessPaths;
+ }
+
+ public Optional<List<TColumnAccessPath>> getDisplayPredicateAccessPaths() {
+ return displayPredicateAccessPaths;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
index 8c0babc39ca..5410de371a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayFirst.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import java.util.List;
@@ -51,4 +52,9 @@ public class ArrayFirst extends ElementAt
public List<FunctionSignature> getImplSignature() {
return SIGNATURES;
}
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitArrayFirst(this, context);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
index e1ed4f4d27b..b9f5650156f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayLast.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import java.util.List;
@@ -51,4 +52,9 @@ public class ArrayLast extends ElementAt
public ElementAt withChildren(List<Expression> children) {
return new ArrayLast(getFunctionParams(children));
}
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitArrayLast(this, context);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
index 5d74f5e1457..1a1a246cea6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ImplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'array_map'.
*/
public class ArrayMap extends ScalarFunction
- implements ImplicitlyCastableSignature, PropagateNullable {
+ implements ImplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0)).args(LambdaType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
index b24d46c98a4..cd2cb974364 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.RewriteWhenAnalyze;
@@ -42,7 +43,8 @@ import java.util.List;
* ScalarFunction 'element_at'. This class is generated by GenerateFunction.
*/
public class ElementAt extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, RewriteWhenAnalyze {
+ implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable,
+ RewriteWhenAnalyze, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
index a929f2dd5d5..e7a054fc864 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.types.LambdaType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
@@ -29,7 +30,7 @@ import java.util.List;
/**
* Interface of HighOrderFunction, provide default FunctionSignature of LambdaType argument
*/
-public interface HighOrderFunction extends ExplicitlyCastableSignature {
+public interface HighOrderFunction extends ExplicitlyCastableSignature, PreferPushDownProject {
@Override
default List<FunctionSignature> getSignatures() {
return new ImmutableList.Builder<FunctionSignature>()
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
index b8ecc8667c7..517bfaa7748 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_entry'.
*/
public class MapContainsEntry extends ScalarFunction
- implements TernaryExpression, ExplicitlyCastableSignature {
+ implements TernaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
/**
* Basic signature, forcing search key/value type to convert to the same type as map's key/value type.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
index 4fc4c32f233..845e84f71b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_key'. This class is generated by GenerateFunction.
*/
public class MapContainsKey extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
index a60f6653cde..fd42c42fb65 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_value'. This class is generated by GenerateFunction.
*/
public class MapContainsValue extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature, PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE = ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
index 4be48638e5e..22a14c31df3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
@@ -41,7 +42,7 @@ import java.util.List;
* fields 'key' and 'value'.
*/
public class MapEntries extends ScalarFunction
- implements UnaryExpression, CustomSignature, PropagateNullable {
+ implements UnaryExpression, CustomSignature, PropagateNullable, PreferPushDownProject {
/**
* constructor with 1 argument.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
index 957dae2844a..a3123d29cb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_keys'.
*/
public class MapKeys extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(MapType.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
index e7ca7f48dd3..1622b277156 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_values'.
*/
public class MapValues extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new FollowToAnyDataType(0))).args(MapType.of(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
index 85bb424de19..ed29818b688 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
@@ -29,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NullType;
+import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import com.google.common.base.Preconditions;
@@ -40,7 +42,7 @@ import java.util.List;
* ScalarFunction 'struct_element'.
*/
public class StructElement extends ScalarFunction
- implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral {
+ implements ExplicitlyCastableSignature, AlwaysNullable, PropagateNullLiteral, PreferPushDownProject {
/**
* constructor with 0 or more arguments.
@@ -98,10 +100,11 @@ public class StructElement extends ScalarFunction
}
} else if (child(1) instanceof StringLikeLiteral) {
String name = ((StringLikeLiteral) child(1)).getStringValue();
- if (!structArgType.getNameToFields().containsKey(name.toLowerCase())) {
+ StructField field = structArgType.getField(name);
+ if (field == null) {
throw new AnalysisException("the specified field name " + name + " was not found: " + this.toSql());
} else {
- retType = structArgType.getNameToFields().get(name).getDataType();
+ retType = field.getDataType();
}
} else {
throw new AnalysisException("struct_element only allows"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index d347352c5ef..d48ba889cd0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -54,10 +54,12 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayEnumerat
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExcept;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayExists;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFilter;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirst;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFirstIndex;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayFlatten;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayIntersect;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayJoin;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
@@ -641,6 +643,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayFilter, context);
}
+ default R visitArrayFirst(ArrayFirst arrayFirst, C context) {
+ return visitElementAt(arrayFirst, context);
+ }
+
default R visitArrayFirstIndex(ArrayFirstIndex arrayFirstIndex, C context) {
return visitScalarFunction(arrayFirstIndex, context);
}
@@ -653,6 +659,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayJoin, context);
}
+ default R visitArrayLast(ArrayLast arrayLast, C context) {
+ return visitElementAt(arrayLast, context);
+ }
+
default R visitArrayLastIndex(ArrayLastIndex arrayLastIndex, C context) {
return visitScalarFunction(arrayLastIndex, context);
}
@@ -757,8 +767,8 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(arrayFlatten, context);
}
- default R visitArrayMap(ArrayMap arraySort, C context) {
- return visitScalarFunction(arraySort, context);
+ default R visitArrayMap(ArrayMap arrayMap, C context) {
+ return visitScalarFunction(arrayMap, context);
}
default R visitArrayMatchAll(ArrayMatchAll arrayMatchAll, C context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 6ae9e5dd32a..860fbee192e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -51,16 +51,17 @@ public class LogicalFileScan extends LogicalCatalogRelation {
protected final Optional<TableSample> tableSample;
protected final Optional<TableSnapshot> tableSnapshot;
protected final Optional<TableScanParams> scanParams;
+ protected final Optional<List<Slot>> cachedOutputs;
public LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
Collection<Slot> operativeSlots,
Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot,
- Optional<TableScanParams> scanParams) {
+ Optional<TableScanParams> scanParams, Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier,
table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)),
operativeSlots, ImmutableList.of(),
tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.empty());
+ scanParams, Optional.empty(), Optional.empty(), cachedOutputs);
}
/**
@@ -71,22 +72,24 @@ public class LogicalFileScan extends LogicalCatalogRelation {
List<NamedExpression> virtualColumns, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot, Optional<TableScanParams> scanParams,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
- String tableAlias) {
+ String tableAlias, Optional<List<Slot>> cachedSlots) {
super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, operativeSlots, virtualColumns,
groupExpression, logicalProperties, tableAlias);
this.selectedPartitions = selectedPartitions;
this.tableSample = tableSample;
this.tableSnapshot = tableSnapshot;
this.scanParams = scanParams;
+ this.cachedOutputs = cachedSlots;
}
protected LogicalFileScan(RelationId id, ExternalTable table, List<String> qualifier,
SelectedPartitions selectedPartitions, Collection<Slot> operativeSlots,
List<NamedExpression> virtualColumns, Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot, Optional<TableScanParams> scanParams,
- Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, logicalProperties, "");
+ scanParams, groupExpression, logicalProperties, "", cachedOutputs);
}
public SelectedPartitions getSelectedPartitions() {
@@ -126,7 +129,7 @@ public class LogicalFileScan extends LogicalCatalogRelation {
public LogicalFileScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, groupExpression, Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
@@ -134,26 +137,26 @@ public class LogicalFileScan extends LogicalCatalogRelation {
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, logicalProperties, tableAlias);
+ scanParams, groupExpression, logicalProperties, tableAlias, cachedOutputs);
}
public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
public LogicalFileScan withRelationId(RelationId relationId) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.empty(), tableAlias);
+ scanParams, Optional.empty(), Optional.empty(), tableAlias, cachedOutputs);
}
public LogicalFileScan withTableAlias(String tableAlias) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias);
+ scanParams, Optional.empty(), Optional.of(getLogicalProperties()), tableAlias, cachedOutputs);
}
@Override
@@ -166,6 +169,19 @@ public class LogicalFileScan extends LogicalCatalogRelation {
return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions);
}
+ @Override
+ public List<Slot> computeOutput() {
+ if (cachedOutputs.isPresent()) {
+ return cachedOutputs.get();
+ }
+ return super.computeOutput();
+ }
+
+ @Override
+ public List<Slot> computeAsteriskOutput() {
+ return super.computeAsteriskOutput();
+ }
+
/**
* SelectedPartitions contains the selected partitions and the total partition number.
* Mainly for hive table partition pruning.
@@ -223,7 +239,13 @@ public class LogicalFileScan extends LogicalCatalogRelation {
public LogicalFileScan withOperativeSlots(Collection<Slot> operativeSlots) {
return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
- scanParams, groupExpression, Optional.of(getLogicalProperties()));
+ scanParams, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
+ }
+
+ public LogicalFileScan withCachedOutput(List<Slot> cachedOutputs) {
+ return new LogicalFileScan(relationId, (ExternalTable) table, qualifier,
+ selectedPartitions, operativeSlots, virtualColumns, tableSample, tableSnapshot,
+ scanParams, groupExpression, Optional.empty(), Optional.of(cachedOutputs));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
index 9a123a04d2b..134f48309d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java
@@ -77,9 +77,10 @@ public class LogicalHudiScan extends LogicalFileScan {
Collection<Slot> operativeSlots,
List<NamedExpression> virtualColumns,
Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties) {
+ Optional<LogicalProperties> logicalProperties,
+ Optional<List<Slot>> cachedOutputs) {
super(id, table, qualifier, selectedPartitions, operativeSlots, virtualColumns,
- tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties);
+ tableSample, tableSnapshot, scanParams, groupExpression, logicalProperties, cachedOutputs);
Objects.requireNonNull(scanParams, "scanParams should not null");
Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null");
this.incrementalRelation = incrementalRelation;
@@ -87,10 +88,11 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalHudiScan(RelationId id, ExternalTable table, List<String> qualifier,
Collection<Slot> operativeSlots, Optional<TableScanParams> scanParams,
- Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot) {
+ Optional<TableSample> tableSample, Optional<TableSnapshot> tableSnapshot,
+ Optional<List<Slot>> cachedOutputs) {
this(id, table, qualifier, ((HMSExternalTable) table).initHudiSelectedPartitions(tableSnapshot),
tableSample, tableSnapshot, scanParams, Optional.empty(), operativeSlots, ImmutableList.of(),
- Optional.empty(), Optional.empty());
+ Optional.empty(), Optional.empty(), cachedOutputs);
}
public Optional<TableScanParams> getScanParams() {
@@ -140,7 +142,7 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalHudiScan withGroupExpression(Optional<GroupExpression> groupExpression) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
@@ -148,20 +150,20 @@ public class LogicalHudiScan extends LogicalFileScan {
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, logicalProperties);
+ operativeSlots, virtualColumns, groupExpression, logicalProperties, cachedOutputs);
}
public LogicalHudiScan withSelectedPartitions(SelectedPartitions selectedPartitions) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
public LogicalHudiScan withRelationId(RelationId relationId) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
@Override
@@ -173,7 +175,7 @@ public class LogicalHudiScan extends LogicalFileScan {
public LogicalFileScan withOperativeSlots(Collection<Slot> operativeSlots) {
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
/**
@@ -226,6 +228,6 @@ public class LogicalHudiScan extends LogicalFileScan {
}
return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier,
selectedPartitions, tableSample, tableSnapshot, scanParams, newIncrementalRelation,
- operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()));
+ operativeSlots, virtualColumns, groupExpression, Optional.of(getLogicalProperties()), cachedOutputs);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 4278c22f123..394e414cc2e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -851,4 +851,21 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
}
return replaceMap;
}
+
+ /** withPrunedTypeSlots */
+ public LogicalOlapScan withPrunedTypeSlots(List<Slot> outputSlots) {
+ Map<Pair<Long, String>, Slot> replaceSlotMap = new HashMap<>();
+ for (Slot outputSlot : outputSlots) {
+ Pair<Long, String> key = Pair.of(selectedIndexId, outputSlot.getName());
+ replaceSlotMap.put(key, outputSlot);
+ }
+
+ return new LogicalOlapScan(relationId, (Table) table, qualifier,
+ groupExpression, Optional.empty(),
+ selectedPartitionIds, partitionPruned, selectedTabletIds,
+ selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions,
+ hints, replaceSlotMap, tableSample, directMvScan, colToSubPathsMap,
+ manuallySpecifiedTabletIds, operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
+ annOrderKeys, annLimit, tableAlias);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
index a434f81b231..c6d0241f393 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java
@@ -47,20 +47,31 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
private final TableValuedFunction function;
private final ImmutableList<String> qualifier;
private final ImmutableList<Slot> operativeSlots;
+ private final Optional<List<Slot>> cachedOutputs;
public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList<Slot> operativeSlots) {
super(id, PlanType.LOGICAL_TVF_RELATION);
this.operativeSlots = operativeSlots;
this.function = function;
- qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.cachedOutputs = Optional.empty();
}
public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList<Slot> operativeSlots,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
+ this(id, function, operativeSlots, Optional.empty(), groupExpression, logicalProperties);
+ }
+
+ public LogicalTVFRelation(RelationId id, TableValuedFunction function,
+ ImmutableList<Slot> operativeSlots,
+ Optional<List<Slot>> cachedOutputs,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties) {
super(id, PlanType.LOGICAL_TVF_RELATION, groupExpression, logicalProperties);
this.operativeSlots = operativeSlots;
this.function = function;
- qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
+ this.cachedOutputs = Objects.requireNonNull(cachedOutputs, "cachedOutputs can not be null");
+ this.qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName());
}
@Override
@@ -120,6 +131,9 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
@Override
public List<Slot> computeOutput() {
+ if (cachedOutputs.isPresent()) {
+ return cachedOutputs.get();
+ }
IdGenerator<ExprId> exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator();
return function.getTable().getBaseSchema()
.stream()
@@ -138,4 +152,9 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation,
public TableValuedFunction getFunction() {
return function;
}
+
+ public LogicalTVFRelation withCachedOutputs(List<Slot> replaceSlots) {
+ return new LogicalTVFRelation(relationId, function, Utils.fastToImmutableList(operativeSlots),
+ Optional.of(replaceSlots), Optional.empty(), Optional.empty());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
index 9a34ab740d3..6d4ec539ff3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
@@ -25,7 +25,7 @@ import java.util.Objects;
/**
* Array type in Nereids.
*/
-public class ArrayType extends DataType implements ComplexDataType {
+public class ArrayType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final ArrayType SYSTEM_DEFAULT = new ArrayType(NullType.INSTANCE, true);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
index fb8346987f7..176c1db1d0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/MapType.java
@@ -27,7 +27,7 @@ import java.util.Objects;
* Struct type in Nereids.
*/
@Developing
-public class MapType extends DataType implements ComplexDataType {
+public class MapType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final MapType SYSTEM_DEFAULT = new MapType();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java
new file mode 100644
index 00000000000..2d5e464f042
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.types;
+
+/** NestedColumnPrunable */
+public interface NestedColumnPrunable {
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
index 88317682c4c..63e94bc369b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructField.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.types;
import org.apache.doris.nereids.util.Utils;
-import java.util.Locale;
import java.util.Objects;
/**
@@ -41,7 +40,7 @@ public class StructField {
* @param nullable Indicates if values of this field can be `null` values
*/
public StructField(String name, DataType dataType, boolean nullable, String comment) {
- this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase(Locale.ROOT);
+ this.name = Objects.requireNonNull(name, "name should not be null").toLowerCase();
this.dataType = Objects.requireNonNull(dataType, "dataType should not be null");
this.nullable = nullable;
this.comment = Objects.requireNonNull(comment, "comment should not be null");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
index ffbff7c61e1..8296ac28281 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java
@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -35,7 +36,7 @@ import java.util.stream.Collectors;
* Struct type in Nereids.
*/
@Developing
-public class StructType extends DataType implements ComplexDataType {
+public class StructType extends DataType implements ComplexDataType, NestedColumnPrunable {
public static final StructType SYSTEM_DEFAULT = new StructType();
@@ -55,7 +56,8 @@ public class StructType extends DataType implements ComplexDataType {
public StructType(List<StructField> fields) {
this.fields = ImmutableList.copyOf(Objects.requireNonNull(fields, "fields should not be null"));
// field name should be lowercase and check the same or not
- this.nameToFields = new HashMap<>();
+ // ATTN: should use LinkedHashMap to keep order
+ this.nameToFields = new LinkedHashMap<>();
for (StructField field : this.fields) {
String fieldName = field.getName().toLowerCase();
StructField existingField = this.nameToFields.put(fieldName, field);
@@ -73,6 +75,10 @@ public class StructType extends DataType implements ComplexDataType {
return nameToFields;
}
+ public StructField getField(String name) {
+ return nameToFields.get(name.toLowerCase());
+ }
+
@Override
public DataType conversion() {
return new StructType(fields.stream().map(StructField::conversion).collect(Collectors.toList()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 98399c93d31..f9c6ed0b237 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -1088,6 +1088,9 @@ public class OlapScanNode extends ScanNode {
output.append(prefix).append("rewrittenProjectList: ").append(
getExplainString(rewrittenProjectList)).append("\n");
}
+
+ printNestedColumns(output, prefix);
+
return output.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 27369e40e18..382782c393a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -518,7 +518,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
return expBuilder.toString();
}
- private String getplanNodeExplainString(String prefix, TExplainLevel detailLevel) {
+ private String getPlanNodeExplainString(String prefix, TExplainLevel detailLevel) {
StringBuilder expBuilder = new StringBuilder();
expBuilder.append(getNodeExplainString(prefix, detailLevel));
if (limit != -1) {
@@ -533,7 +533,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
}
public void getExplainStringMap(TExplainLevel detailLevel, Map<Integer, String> planNodeMap) {
- planNodeMap.put(id.asInt(), getplanNodeExplainString("", detailLevel));
+ planNodeMap.put(id.asInt(), getPlanNodeExplainString("", detailLevel));
for (int i = 0; i < children.size(); ++i) {
children.get(i).getExplainStringMap(detailLevel, planNodeMap);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index fee9b0f8afc..ee8270fbd79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -47,10 +47,13 @@ import org.apache.doris.datasource.FederationBackendPolicy;
import org.apache.doris.datasource.SplitAssignment;
import org.apache.doris.datasource.SplitGenerator;
import org.apache.doris.datasource.SplitSource;
+import org.apache.doris.datasource.iceberg.source.IcebergScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.statistics.StatisticalType;
import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPlanNode;
import org.apache.doris.thrift.TScanRange;
@@ -65,6 +68,7 @@ import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
import com.google.common.collect.TreeRangeSet;
import org.apache.commons.collections.map.CaseInsensitiveMap;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -539,6 +543,111 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
.addValue(super.debugString()).toString();
}
+ protected void printNestedColumns(StringBuilder output, String prefix) {
+ boolean printNestedColumnsHeader = true;
+ for (SlotDescriptor slot : getTupleDesc().getSlots()) {
+ String prunedType = null;
+ if (!slot.getType().equals(slot.getColumn().getType())) {
+ prunedType = slot.getType().toString();
+ }
+ String displayAllAccessPathsString = null;
+ if (slot.getDisplayAllAccessPaths() != null
+ && slot.getDisplayAllAccessPaths() != null
+ && !slot.getDisplayAllAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayAllAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getAllAccessPaths(),
+ slot.getDisplayAllAccessPaths()
+ );
+ } else {
+ displayAllAccessPathsString = slot.getDisplayAllAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+ String displayPredicateAccessPathsString = null;
+ if (slot.getDisplayPredicateAccessPaths() != null
+ && slot.getDisplayPredicateAccessPaths() != null
+ && !slot.getDisplayPredicateAccessPaths().isEmpty()) {
+ if (this instanceof IcebergScanNode) {
+ displayPredicateAccessPathsString = mergeIcebergAccessPathsWithId(
+ slot.getPredicateAccessPaths(),
+ slot.getDisplayPredicateAccessPaths()
+ );
+ } else {
+ displayPredicateAccessPathsString = slot.getPredicateAccessPaths()
+ .stream()
+ .map(a -> {
+ if (a.type == TAccessPathType.DATA) {
+ return StringUtils.join(a.data_access_path.path, ".");
+ } else {
+ return StringUtils.join(a.meta_access_path.path, ".");
+ }
+ })
+ .collect(Collectors.joining(", "));
+ }
+ }
+
+
+ if (prunedType == null
+ && displayAllAccessPathsString == null
+ && displayPredicateAccessPathsString == null) {
+ continue;
+ }
+
+ if (printNestedColumnsHeader) {
+ output.append(prefix).append("nested columns:\n");
+ printNestedColumnsHeader = false;
+ }
+ output.append(prefix).append(" ").append(slot.getColumn().getName()).append(":\n");
+ output.append(prefix).append(" origin type: ").append(slot.getColumn().getType()).append("\n");
+ if (prunedType != null) {
+ output.append(prefix).append(" pruned type: ").append(prunedType).append("\n");
+ }
+ if (displayAllAccessPathsString != null) {
+ output.append(prefix).append(" all access paths: [")
+ .append(displayAllAccessPathsString).append("]\n");
+ }
+ if (displayPredicateAccessPathsString != null) {
+ output.append(prefix).append(" predicate access paths: [")
+ .append(displayPredicateAccessPathsString).append("]\n");
+ }
+ }
+ }
+
+ private String mergeIcebergAccessPathsWithId(
+ List<TColumnAccessPath> accessPaths, List<TColumnAccessPath> displayAccessPaths) {
+ List<String> mergeDisplayAccessPaths = Lists.newArrayList();
+ for (int i = 0; i < displayAccessPaths.size(); i++) {
+ TColumnAccessPath displayAccessPath = displayAccessPaths.get(i);
+ TColumnAccessPath idAccessPath = accessPaths.get(i);
+ List<String> nameAccessPathStrings = displayAccessPath.type == TAccessPathType.DATA
+ ? displayAccessPath.data_access_path.path : displayAccessPath.meta_access_path.path;
+ List<String> idAccessPathStrings = idAccessPath.type == TAccessPathType.DATA
+ ? idAccessPath.data_access_path.path : idAccessPath.meta_access_path.path;
+
+ List<String> mergedPath = new ArrayList<>();
+ for (int j = 0; j < idAccessPathStrings.size(); j++) {
+ String name = nameAccessPathStrings.get(j);
+ String id = idAccessPathStrings.get(j);
+ if (name.equals(id)) {
+ mergedPath.add(name);
+ } else {
+ mergedPath.add(name + "(" + id + ")");
+ }
+ }
+ mergeDisplayAccessPaths.add(StringUtils.join(mergedPath, "."));
+ }
+ return StringUtils.join(mergeDisplayAccessPaths, ", ");
+ }
+
public List<TupleId> getOutputTupleIds() {
if (outputTupleDesc != null) {
return Lists.newArrayList(outputTupleDesc.getId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 52b1d90ac26..7a06cbc7823 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -409,6 +409,8 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_RUNTIME_FILTER_PARTITION_PRUNE =
"enable_runtime_filter_partition_prune";
+ public static final String ENABLE_PRUNE_NESTED_COLUMN = "enable_prune_nested_column";
+
static final String SESSION_CONTEXT = "session_context";
public static final String DEFAULT_ORDER_BY_LIMIT = "default_order_by_limit";
@@ -1557,6 +1559,13 @@ public class SessionVariable implements Serializable, Writable {
varType = VariableAnnotation.EXPERIMENTAL)
public int topNLazyMaterializationThreshold = 1024;
+ @VariableMgr.VarAttr(name = ENABLE_PRUNE_NESTED_COLUMN, needForward = true,
+ fuzzy = false,
+ varType = VariableAnnotation.EXPERIMENTAL,
+ description = {"是否裁剪map/struct类型", "Whether to prune the type of map/struct"}
+ )
+ public boolean enablePruneNestedColumns = true;
+
public boolean enableTopnLazyMaterialization() {
return ConnectContext.get() != null
&& ConnectContext.get().getSessionVariable().topNLazyMaterializationThreshold > 0;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
new file mode 100644
index 00000000000..8e8d7c44d16
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -0,0 +1,625 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.Triple;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.PlanFragment;
+import org.apache.doris.thrift.TAccessPathType;
+import org.apache.doris.thrift.TColumnAccessPath;
+import org.apache.doris.thrift.TDataAccessPath;
+import org.apache.doris.thrift.TMetaAccessPath;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeSet;
+import java.util.function.Consumer;
+
+public class PruneNestedColumnTest extends TestWithFeService implements MemoPatternMatchSupported {
+ @BeforeAll
+ public void createTable() throws Exception {
+ createDatabase("test");
+ useDatabase("test");
+
+ createTable("create table tbl(\n"
+ + " id int,\n"
+ + " s struct<\n"
+ + " city: string,\n"
+ + " data: array<map<\n"
+ + " int,\n"
+ + " struct<a: int, b: double>\n"
+ + " >>\n"
+ + ">)\n"
+ + "properties ('replication_num'='1')");
+
+ createTable("create table tbl2(\n"
+ + " id2 int,\n"
+ + " s2 struct<\n"
+ + " city2: string,\n"
+ + " data2: array<map<\n"
+ + " int,\n"
+ + " struct<a2: int, b2: double>\n"
+ + " >>\n"
+ + ">)\n"
+ + "properties ('replication_num'='1')");
+
+ connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name());
+ connectContext.getSessionVariable().enableNereidsTimeout = false;
+ }
+
+ @Test
+ public void testCaseInsensitive() throws Exception {
+ assertColumn("select struct_element(MAP_VALUES(struct_element(S, 'DATA')[1])[1], 'B') from tbl",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testMap() throws Exception {
+ assertColumn("select MAP_KEYS(struct_element(s, 'data')[0])[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select MAP_VALUES(struct_element(s, 'data')[0])[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testStruct() throws Throwable {
+ assertColumn("select struct_element(s, 1) from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select struct_element(map_values(struct_element(s, 'data')[0])[0], 1) from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testPruneCast() throws Exception {
+ assertColumn("select struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:int,y:double>>>>), 'k') from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select struct_element(map_values(struct_element(cast(s as struct<k:text,l:array<map<int,struct<x:double,y:double>>>>), 'l')[0])[0], 'x') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+
+ assertColumns("select struct_element(s, 'city') from (select * from tbl union all select * from tbl2)t",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city2:text>",
+ ImmutableList.of(path("s2", "city2")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ )
+ )
+ );
+
+ assertColumns("select struct_element(s, 'city'), struct_element(map_values(struct_element(s, 'data')[0])[0], 'b') from (select * from tbl union all select * from tbl2)t",
+ ImmutableList.of(
+ Triple.of(
+ "struct<city2:text,data2:array<map<int,struct<b2:double>>>>",
+ ImmutableList.of(path("s2", "city2"), path("s2", "data2", "*", "VALUES", "b2")),
+ ImmutableList.of()
+ ),
+ Triple.of(
+ "struct<city:text,data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "city"), path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testPruneArrayLambda() throws Exception {
+ // map_values(struct_element(s, 'data').*)[0].a
+ assertColumn("select struct_element(array_map(x -> map_values(x)[0], struct_element(s, 'data'))[0], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select array_map((x, y) -> struct_element(map_values(x)[0], 'a') + struct_element(map_values(y)[0], 'b'), struct_element(s, 'data'), struct_element(s, 'data')) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a"), path("s", "data", "*", "VALUES", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testProject() throws Exception {
+ assertColumn("select 100 from tbl", null, null, null);
+ assertColumn("select * from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select test.tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select internal.test.tbl.* from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select s from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'city'), s from tbl",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'city') from tbl",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data') from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data')[1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*")),
+ ImmutableList.of()
+ );
+ assertColumn("select map_keys(struct_element(s, 'data')[1]) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of()
+ );
+ assertColumn("select map_values(struct_element(s, 'data')[1]) from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(map_values(struct_element(s, 'data')[1])[1], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(s, 'data')[1][1] from tbl",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(struct_element(s, 'data')[1][1], 'a') from tbl",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "a")),
+ ImmutableList.of()
+ );
+ assertColumn("select struct_element(struct_element(s, 'data')[1][1], 'b') from tbl",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "b")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testFilter() throws Throwable {
+ assertColumn("select 100 from tbl where s is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s"))
+ );
+
+ assertColumn("select 100 from tbl where if(id = 1, null, s) is not null and struct_element(s, 'city') = 'beijing'",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s"))
+ );
+
+ assertColumn("select 100 from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select 100 from tbl where struct_element(s, 'data') is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data")),
+ ImmutableList.of(path("s", "data"))
+ );
+ assertColumn("select 100 from tbl where struct_element(s, 'data')[1] is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*")),
+ ImmutableList.of(path("s", "data", "*"))
+ );
+ assertColumn("select 100 from tbl where map_keys(struct_element(s, 'data')[1]) is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "KEYS")),
+ ImmutableList.of(path("s", "data", "*", "KEYS"))
+ );
+ assertColumn("select 100 from tbl where map_values(struct_element(s, 'data')[1]) is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES")),
+ ImmutableList.of(path("s", "data", "*", "VALUES"))
+ );
+ assertColumn("select 100 from tbl where struct_element(map_values(struct_element(s, 'data')[1])[1], 'a') is not null",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a")),
+ ImmutableList.of(path("s", "data", "*", "VALUES", "a"))
+ );
+ assertColumn("select 100 from tbl where struct_element(s, 'data')[1][1] is not null",
+ "struct<data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*")),
+ ImmutableList.of(path("s", "data", "*", "*"))
+ );
+ assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'a') is not null",
+ "struct<data:array<map<int,struct<a:int>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "a")),
+ ImmutableList.of(path("s", "data", "*", "*", "a"))
+ );
+ assertColumn("select 100 from tbl where struct_element(struct_element(s, 'data')[1][1], 'b') is not null",
+ "struct<data:array<map<int,struct<b:double>>>>",
+ ImmutableList.of(path("s", "data", "*", "*", "b")),
+ ImmutableList.of(path("s", "data", "*", "*", "b"))
+ );
+ }
+
+ @Test
+ public void testProjectFilter() throws Throwable {
+ assertColumn("select s from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data"), path("s", "city")),
+ ImmutableList.of(path("s", "city"))
+ );
+
+ assertColumn("select struct_element(s, 'data') from tbl where struct_element(s, 'city') is not null and struct_element(s, 'data') is not null",
+ "struct<city:text,data:array<map<int,struct<a:int,b:double>>>>",
+ ImmutableList.of(path("s", "data"), path("s", "city")),
+ ImmutableList.of(path("s", "data"), path("s", "city"))
+ );
+ }
+
+ @Test
+ public void testCte() throws Throwable {
+ assertColumn("with t as (select id, s from tbl) select struct_element(t1.s, 'city') from t t1 join t t2 on t1.id = t2.id",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("with t as (select id, struct_element(s, 'city') as c from tbl) select t1.c from t t1 join t t2 on t1.id = t2.id",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testUnion() throws Throwable {
+ assertColumn("select struct_element(s, 'city') from (select s from tbl union all select null)a",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("select * from (select struct_element(s, 'city') from tbl union all select null)a",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testCteAndUnion() throws Throwable {
+ assertColumn("with t as (select id, s from tbl) select struct_element(s, 'city') from (select * from t union all select 1, null) tmp",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+
+ assertColumn("with t as (select id, s from tbl) select * from (select struct_element(s, 'city') from t union all select null) tmp",
+ "struct<city:text>",
+ ImmutableList.of(path("s", "city")),
+ ImmutableList.of()
+ );
+ }
+
+ @Test
+ public void testPushDownThroughJoin() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select * from tbl)a join (select 100 id, 'f1' name)b on a.id=b.id")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalProject(
+ logicalJoin(
+ logicalProject(
+ logicalFilter(
+ logicalOlapScan()
+ )
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ }),
+ logicalOneRowRelation()
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 1 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughWindow() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city'), r from (select s, rank() over(partition by id) r from tbl t)a")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalProject(
+ logicalWindow(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ })
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughPartitionTopN() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city'), r from (select s, rank() over(partition by id) r from tbl t limit 10)a")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalLimit(
+ logicalLimit(
+ logicalProject(
+ logicalWindow(
+ logicalPartitionTopN(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(2, p.getProjects().size());
+ Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias && o.child(0) instanceof StructElement));
+ return true;
+ })
+ )
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 2 && p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ )
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughUnion() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select id, s from tbl union all select 1, null) tmp")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalUnion(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(1, p.getProjects().size());
+ Assertions.assertInstanceOf(StructElement.class, p.getProjects().get(0).child(0));
+ return true;
+ })
+ ).when(u -> {
+ Assertions.assertEquals(1, u.getConstantExprsList().size());
+ Assertions.assertInstanceOf(NullLiteral.class, u.getConstantExprsList().get(0).get(0).child(0));
+ return true;
+ })
+ )
+ );
+ }
+
+ private void assertColumn(String sql, String expectType,
+ List<TColumnAccessPath> expectAllAccessPaths,
+ List<TColumnAccessPath> expectPredicateAccessPaths) throws Exception {
+ assertColumns(sql, expectType == null ? null : ImmutableList.of(Triple.of(expectType, expectAllAccessPaths, expectPredicateAccessPaths)));
+ }
+
+ private void assertColumns(String sql,
+ List<Triple<String, List<TColumnAccessPath>, List<TColumnAccessPath>>> expectResults) throws Exception {
+ Pair<PhysicalPlan, List<SlotDescriptor>> result = collectComplexSlots(sql);
+ PhysicalPlan physicalPlan = result.first;
+ List<SlotDescriptor> slotDescriptors = result.second;
+ if (expectResults == null) {
+ Assertions.assertEquals(0, slotDescriptors.size());
+ return;
+ }
+
+ Assertions.assertEquals(expectResults.size(), slotDescriptors.size());
+ int slotIndex = 0;
+ for (Triple<String, List<TColumnAccessPath>, List<TColumnAccessPath>> expectResult : expectResults) {
+ String expectType = expectResult.left;
+ List<TColumnAccessPath> expectAllAccessPaths = expectResult.middle;
+ List<TColumnAccessPath> expectPredicateAccessPaths = expectResult.right;
+ SlotDescriptor slotDescriptor = slotDescriptors.get(slotIndex++);
+ Assertions.assertEquals(expectType, slotDescriptor.getType().toString());
+
+ TreeSet<TColumnAccessPath> expectAllAccessPathSet = new TreeSet<>(expectAllAccessPaths);
+ TreeSet<TColumnAccessPath> actualAllAccessPaths
+ = new TreeSet<>(slotDescriptor.getAllAccessPaths());
+ Assertions.assertEquals(expectAllAccessPathSet, actualAllAccessPaths);
+
+ TreeSet<TColumnAccessPath> expectPredicateAccessPathSet = new TreeSet<>(expectPredicateAccessPaths);
+ TreeSet<TColumnAccessPath> actualPredicateAccessPaths
+ = new TreeSet<>(slotDescriptor.getPredicateAccessPaths());
+ Assertions.assertEquals(expectPredicateAccessPathSet, actualPredicateAccessPaths);
+
+ Map<Integer, DataType> slotIdToDataTypes = new LinkedHashMap<>();
+ Consumer<Expression> assertHasSameType = e -> {
+ if (e instanceof NamedExpression) {
+ DataType dataType = slotIdToDataTypes.get(((NamedExpression) e).getExprId().asInt());
+ if (dataType != null) {
+ Assertions.assertEquals(dataType, e.getDataType());
+ } else {
+ slotIdToDataTypes.put(((NamedExpression) e).getExprId().asInt(), e.getDataType());
+ }
+ }
+ };
+
+ // assert same slot id has same type
+ physicalPlan.foreachUp(plan -> {
+ List<? extends Expression> expressions = ((PhysicalPlan) plan).getExpressions();
+ for (Expression expression : expressions) {
+ expression.foreach(e -> {
+ assertHasSameType.accept((Expression) e);
+ if (e instanceof Alias && e.child(0) instanceof Slot) {
+ assertHasSameType.accept((Alias) e);
+ } else if (e instanceof ArrayItemReference) {
+ assertHasSameType.accept((ArrayItemReference) e);
+ }
+ });
+ }
+
+ if (plan instanceof PhysicalCTEConsumer) {
+ for (Entry<Slot, Collection<Slot>> kv : ((PhysicalCTEConsumer) plan).getProducerToConsumerSlotMap()
+ .asMap().entrySet()) {
+ Slot producerSlot = kv.getKey();
+ for (Slot consumerSlot : kv.getValue()) {
+ Assertions.assertEquals(producerSlot.getDataType(), consumerSlot.getDataType());
+ }
+ }
+ } else if (plan instanceof PhysicalUnion) {
+ List<Slot> output = ((PhysicalUnion) plan).getOutput();
+ for (List<SlotReference> regularChildrenOutput : ((PhysicalUnion) plan).getRegularChildrenOutputs()) {
+ Assertions.assertEquals(output.size(), regularChildrenOutput.size());
+ for (int i = 0; i < output.size(); i++) {
+ Assertions.assertEquals(output.get(i).getDataType(), regularChildrenOutput.get(i).getDataType());
+ }
+ }
+ }
+ });
+ }
+ }
+
+ private Pair<PhysicalPlan, List<SlotDescriptor>> collectComplexSlots(String sql) throws Exception {
+ NereidsPlanner planner = (NereidsPlanner) executeNereidsSql(sql).planner();
+ List<SlotDescriptor> complexSlots = new ArrayList<>();
+ PhysicalPlan physicalPlan = planner.getPhysicalPlan();
+ for (PlanFragment fragment : planner.getFragments()) {
+ List<OlapScanNode> olapScanNodes = fragment.getPlanRoot().collectInCurrentFragment(OlapScanNode.class::isInstance);
+ for (OlapScanNode olapScanNode : olapScanNodes) {
+ List<SlotDescriptor> slots = olapScanNode.getTupleDesc().getSlots();
+ for (SlotDescriptor slot : slots) {
+ Type type = slot.getType();
+ if (type.isComplexType() || type.isVariantType()) {
+ complexSlots.add(slot);
+ }
+ }
+ }
+ }
+ return Pair.of(physicalPlan, complexSlots);
+ }
+
+ private TColumnAccessPath path(String... path) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.DATA);
+ accessPath.data_access_path = new TDataAccessPath(ImmutableList.copyOf(path));
+ return accessPath;
+ }
+
+ private TColumnAccessPath metaPath(String... path) {
+ TColumnAccessPath accessPath = new TColumnAccessPath(TAccessPathType.META);
+ accessPath.meta_access_path = new TMetaAccessPath(ImmutableList.copyOf(path));
+ return accessPath;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 21c45c7135c..55d8e70d50d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -559,8 +559,8 @@ public abstract class TestWithFeService {
&& connectContext.getState().getErrorCode() == null) {
return stmtExecutor;
} else {
- // throw new IllegalStateException(connectContext.getState().getErrorMessage());
- return null;
+ throw new IllegalStateException(connectContext.getState().getErrorMessage());
+ // return null;
}
}
@@ -574,7 +574,7 @@ public abstract class TestWithFeService {
}
}
- public void executeNereidsSql(String queryStr) throws Exception {
+ public StmtExecutor executeNereidsSql(String queryStr) throws Exception {
connectContext.getState().reset();
StatementContext statementContext = new StatementContext(connectContext, new OriginStatement(queryStr, 0));
@@ -590,6 +590,7 @@ public abstract class TestWithFeService {
|| connectContext.getState().getErrorCode() != null) {
throw new IllegalStateException(connectContext.getState().getErrorMessage());
}
+ return stmtExecutor;
}
public void createDatabase(String db) throws Exception {
diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift
index a265db13e5d..23f72d4bc68 100644
--- a/gensrc/thrift/Descriptors.thrift
+++ b/gensrc/thrift/Descriptors.thrift
@@ -27,6 +27,49 @@ enum TPatternType {
MATCH_NAME_GLOB = 2
}
+enum TAccessPathType {
+ DATA = 1,
+ META = 2 // use to prune `where s.data is not null` by only scan the meta of s.data
+}
+
+struct TDataAccessPath {
+ // the specification of special path:
+ // <empty>: access the whole complex column
+ // *:
+ // 1. access every items when the type is array
+ // 2. access key and value when the type is map
+ // KEYS: only access the keys of map
+ // VALUES: only access the keys of map
+ //
+ // example:
+ // s: struct<
+ // data: array<
+ // map<
+ // int,
+ // struct<
+ // a: id
+ // b: double
+ // >
+ // >
+ // >
+ // >
+ // if we want to access `map_keys(s.data[0])`, the path will be: ['s', 'data', '*', 'KEYS'],
+ // if we want to access `map_values(s.data[0])[0].b`, the path will be: ['s', 'data', '*', 'VALUES', 'b'],
+ // if we want to access `s.data[0]['k'].b`, the path will be ['s', 'data', '*', '*', 'b']
+ // if we want to access the whole struct of s, the path will be: ['s'],
+ 1: required list<string> path
+}
+
+struct TMetaAccessPath {
+ 1: required list<string> path
+}
+
+struct TColumnAccessPath {
+ 1: required TAccessPathType type
+ 2: optional TDataAccessPath data_access_path
+ 3: optional TMetaAccessPath meta_access_path
+}
+
struct TColumn {
1: required string column_name
2: required Types.TColumnType column_type
@@ -69,14 +112,17 @@ struct TSlotDescriptor {
11: optional i32 col_unique_id = -1
12: optional bool is_key = false
// If set to false, then such slots will be ignored during
- // materialize them.Used to optmize to read less data and less memory usage
+ // materialize them.Used to optimize to read less data and less memory usage
13: optional bool need_materialize = true
14: optional bool is_auto_increment = false;
// subcolumn path info list for semi structure column(variant)
+ // deprecated: will be replaced to column_access_paths
15: optional list<string> column_paths
16: optional string col_default_value
17: optional Types.TPrimitiveType primitive_type = Types.TPrimitiveType.INVALID_TYPE
18: optional Exprs.TExpr virtual_column_expr
+ 19: optional list<TColumnAccessPath> all_access_paths
+ 20: optional list<TColumnAccessPath> predicate_access_paths
}
struct TTupleDescriptor {
diff --git a/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
new file mode 100644
index 00000000000..74f336d27e5
--- /dev/null
+++ b/regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
@@ -0,0 +1,24 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1 {"city":"beijing", "data":[{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]}
+2 {"city":"shanghai", "data":[{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]}
+
+-- !sql1 --
+1 [10]
+
+-- !sql2 --
+1 beijing
+2 shanghai
+
+-- !sql3 --
+1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]
+2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]
+
+-- !sql4 --
+1 [{1:{"a":10, "b":20}, 2:{"a":30, "b":40}}]
+2 [{2:{"a":50, "b":40}, 1:{"a":70, "b":80}}]
+
+-- !sql5 --
+1 beijing
+2 shanghai
+
diff --git a/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
new file mode 100644
index 00000000000..ad032fab8f2
--- /dev/null
+++ b/regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_pruned_columns") {
+ sql """DROP TABLE IF EXISTS `tbl_test_pruned_columns`"""
+ sql """
+ CREATE TABLE `tbl_test_pruned_columns` (
+ `id` int NULL,
+ `s` struct<city:text,data:array<map<int,struct<a:int,b:double>>>> NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ DISTRIBUTED BY RANDOM BUCKETS AUTO
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into `tbl_test_pruned_columns` values
+ (1, named_struct('city', 'beijing', 'data', array(map(1, named_struct('a', 10, 'b', 20.0), 2, named_struct('a', 30, 'b', 40))))),
+ (2, named_struct('city', 'shanghai', 'data', array(map(2, named_struct('a', 50, 'b', 40.0), 1, named_struct('a', 70, 'b', 80)))));
+ """
+
+ qt_sql """
+ select * from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql1 """
+ select b.id, array_map(x -> struct_element(map_values(x)[1], 'a'), struct_element(s, 'data')) from `tbl_test_pruned_columns` t join (select 1 id) b on t.id = b.id order by 1;
+ """
+
+ qt_sql2 """
+ select id, struct_element(s, 'city') from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql3 """
+ select id, struct_element(s, 'data') from `tbl_test_pruned_columns` order by 1;
+ """
+
+ qt_sql4 """
+ select id, struct_element(s, 'data') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1;
+ """
+
+ qt_sql5 """
+ select id, struct_element(s, 'city') from `tbl_test_pruned_columns` where struct_element(struct_element(s, 'data')[1][2], 'b') = 40 order by 1;
+ """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org