You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mr...@apache.org on 2025/10/27 06:32:57 UTC
(doris) branch nested_column_prune updated: Support new thrift protocol(data/meta)
This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/nested_column_prune by this push:
new 79b18201e23 Support new thrift protocol(data/meta)
79b18201e23 is described below
commit 79b18201e23e94c48ffb888749df296f82353e1d
Author: Hu Shenggang <hu...@selectdb.com>
AuthorDate: Mon Oct 27 14:32:01 2025 +0800
Support new thrift protocol(data/meta)
---
be/src/olap/iterators.h | 2 +
be/src/olap/rowset/rowset_reader_context.h | 2 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 119 ++++++++++-----------
be/src/olap/rowset/segment_v2/column_reader.h | 4 +-
be/src/olap/rowset/segment_v2/segment.cpp | 12 +--
be/src/runtime/descriptors.h | 2 +
be/src/vec/exec/scan/olap_scanner.cpp | 6 +-
.../olap/rowset/segment_v2/column_reader_test.cpp | 14 +--
8 files changed, 79 insertions(+), 82 deletions(-)
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index f84a25c3720..bb14f7fc2b4 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -47,6 +47,8 @@ namespace segment_v2 {
struct SubstreamIterator;
}
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class StorageReadOptions {
public:
struct KeyRange {
diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h
index 7caec564af9..f3b4f9eb597 100644
--- a/be/src/olap/rowset/rowset_reader_context.h
+++ b/be/src/olap/rowset/rowset_reader_context.h
@@ -18,6 +18,8 @@
#ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
#define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
+#include <vector>
+
#include "io/io_common.h"
#include "olap/column_predicate.h"
#include "olap/olap_common.h"
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index fff3c6c3f42..cd00f58954a 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -929,26 +929,25 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
const TColumnAccessPaths& access_paths) {
TColumnAccessPaths sub_access_paths = access_paths;
- for (auto it = sub_access_paths.name_access_paths.begin();
- it != sub_access_paths.name_access_paths.end();) {
- TColumnNameAccessPath& name_path = *it;
- if (name_path.path.empty()) {
+ for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) {
+ TColumnAccessPath& name_path = *it;
+ if (name_path.data_access_path.path.empty()) {
return ResultError(
Status::InternalError("Invalid access path for struct column: path is empty"));
}
- if (name_path.path[0] != _column_name) {
+ if (name_path.data_access_path.path[0] != _column_name) {
return ResultError(Status::InternalError(
R"(Invalid access path for column: expected name "{}", got "{}")", _column_name,
- name_path.path[0]));
+ name_path.data_access_path.path[0]));
}
- name_path.path.erase(name_path.path.begin());
- if (!name_path.path.empty()) {
+ name_path.data_access_path.path.erase(name_path.data_access_path.path.begin());
+ if (!name_path.data_access_path.path.empty()) {
++it;
} else {
set_need_to_read();
- it = sub_access_paths.name_access_paths.erase(it);
+ it = sub_access_paths.erase(it);
}
}
return sub_access_paths;
@@ -1091,11 +1090,11 @@ void MapFileColumnIterator::remove_pruned_sub_iterators() {
Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
const TColumnAccessPaths& predicate_access_paths) {
- if (all_access_paths.name_access_paths.empty()) {
+ if (all_access_paths.empty()) {
return Status::OK();
}
- if (!predicate_access_paths.name_access_paths.empty()) {
+ if (!predicate_access_paths.empty()) {
set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
LOG(INFO) << "Map column iterator set sub-column " << _column_name
<< " to READING_FOR_PREDICATE";
@@ -1104,7 +1103,7 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc
auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
- if (sub_all_access_paths.name_access_paths.empty()) {
+ if (sub_all_access_paths.empty()) {
return Status::OK();
}
@@ -1113,35 +1112,35 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc
TColumnAccessPaths key_predicate_access_paths;
TColumnAccessPaths val_predicate_access_paths;
- for (auto paths : sub_all_access_paths.name_access_paths) {
- if (paths.path[0] == "*") {
- paths.path[0] = _key_iterator->column_name();
- key_all_access_paths.name_access_paths.push_back(paths);
- paths.path[0] = _val_iterator->column_name();
- val_all_access_paths.name_access_paths.push_back(paths);
- } else if (paths.path[0] == "KEYS") {
- paths.path[0] = _key_iterator->column_name();
- key_all_access_paths.name_access_paths.push_back(paths);
- } else if (paths.path[0] == "VALUES") {
- paths.path[0] = _val_iterator->column_name();
- val_all_access_paths.name_access_paths.push_back(paths);
+ for (auto paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_all_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_all_access_paths.emplace_back(paths);
}
}
- const auto need_read_keys = !key_all_access_paths.name_access_paths.empty();
- const auto need_read_values = !val_all_access_paths.name_access_paths.empty();
-
- for (auto paths : sub_predicate_access_paths.name_access_paths) {
- if (paths.path[0] == "*") {
- paths.path[0] = _key_iterator->column_name();
- key_predicate_access_paths.name_access_paths.push_back(paths);
- paths.path[0] = _val_iterator->column_name();
- val_predicate_access_paths.name_access_paths.push_back(paths);
- } else if (paths.path[0] == "KEYS") {
- paths.path[0] = _key_iterator->column_name();
- key_predicate_access_paths.name_access_paths.push_back(paths);
- } else if (paths.path[0] == "VALUES") {
- paths.path[0] = _val_iterator->column_name();
- val_predicate_access_paths.name_access_paths.push_back(paths);
+ const auto need_read_keys = !key_all_access_paths.empty();
+ const auto need_read_values = !val_all_access_paths.empty();
+
+ for (auto paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == "*") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "KEYS") {
+ paths.data_access_path.path[0] = _key_iterator->column_name();
+ key_predicate_access_paths.emplace_back(paths);
+ } else if (paths.data_access_path.path[0] == "VALUES") {
+ paths.data_access_path.path[0] = _val_iterator->column_name();
+ val_predicate_access_paths.emplace_back(paths);
}
}
@@ -1290,11 +1289,11 @@ void StructFileColumnIterator::remove_pruned_sub_iterators() {
Status StructFileColumnIterator::set_access_paths(
const TColumnAccessPaths& all_access_paths,
const TColumnAccessPaths& predicate_access_paths) {
- if (all_access_paths.name_access_paths.empty()) {
+ if (all_access_paths.empty()) {
return Status::OK();
}
- if (!predicate_access_paths.name_access_paths.empty()) {
+ if (!predicate_access_paths.empty()) {
set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
LOG(INFO) << "Struct column iterator set sub-column " << _column_name
<< " to READING_FOR_PREDICATE";
@@ -1302,20 +1301,20 @@ Status StructFileColumnIterator::set_access_paths(
auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
- const auto no_sub_column_to_skip = sub_all_access_paths.name_access_paths.empty();
- const auto no_predicate_sub_column = sub_predicate_access_paths.name_access_paths.empty();
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
for (auto& sub_iterator : _sub_column_iterators) {
const auto name = sub_iterator->column_name();
bool need_to_read = no_sub_column_to_skip;
TColumnAccessPaths sub_all_access_paths_of_this;
if (!need_to_read) {
- for (const auto& paths : sub_all_access_paths.name_access_paths) {
- if (paths.path[0] == name) {
- sub_all_access_paths_of_this.name_access_paths.push_back(paths);
+ for (const auto& paths : sub_all_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_all_access_paths_of_this.emplace_back(paths);
}
}
- need_to_read = !sub_all_access_paths_of_this.name_access_paths.empty();
+ need_to_read = !sub_all_access_paths_of_this.empty();
}
if (!need_to_read) {
@@ -1330,9 +1329,9 @@ Status StructFileColumnIterator::set_access_paths(
TColumnAccessPaths sub_predicate_access_paths_of_this;
if (!no_predicate_sub_column) {
- for (const auto& paths : sub_predicate_access_paths.name_access_paths) {
- if (paths.path[0] == name) {
- sub_predicate_access_paths_of_this.name_access_paths.push_back(paths);
+ for (const auto& paths : sub_predicate_access_paths) {
+ if (paths.data_access_path.path[0] == name) {
+ sub_predicate_access_paths_of_this.emplace_back(paths);
}
}
}
@@ -1531,11 +1530,11 @@ void ArrayFileColumnIterator::remove_pruned_sub_iterators() {
Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
const TColumnAccessPaths& predicate_access_paths) {
- if (all_access_paths.name_access_paths.empty()) {
+ if (all_access_paths.empty()) {
return Status::OK();
}
- if (!predicate_access_paths.name_access_paths.empty()) {
+ if (!predicate_access_paths.empty()) {
set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
LOG(INFO) << "Array column iterator set sub-column " << _column_name
<< " to READING_FOR_PREDICATE";
@@ -1544,21 +1543,21 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a
auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
- const auto no_sub_column_to_skip = sub_all_access_paths.name_access_paths.empty();
- const auto no_predicate_sub_column = sub_predicate_access_paths.name_access_paths.empty();
+ const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+ const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
if (!no_sub_column_to_skip) {
- for (auto& path : sub_all_access_paths.name_access_paths) {
- if (path.path[0] == "*") {
- path.path[0] = _item_iterator->column_name();
+ for (auto& path : sub_all_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
}
}
}
if (!no_predicate_sub_column) {
- for (auto& path : sub_predicate_access_paths.name_access_paths) {
- if (path.path[0] == "*") {
- path.path[0] = _item_iterator->column_name();
+ for (auto& path : sub_predicate_access_paths) {
+ if (path.data_access_path.path[0] == "*") {
+ path.data_access_path.path[0] = _item_iterator->column_name();
}
}
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 0abd06f937e..fddbc94c213 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -70,6 +70,8 @@ class FileReader;
struct Slice;
struct StringRef;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
namespace segment_v2 {
class EncodingInfo;
@@ -368,7 +370,7 @@ public:
virtual Status set_access_paths(const TColumnAccessPaths& all_access_paths,
const TColumnAccessPaths& predicate_access_paths) {
- if (!predicate_access_paths.name_access_paths.empty()) {
+ if (!predicate_access_paths.empty()) {
_reading_flag = ReadingFlag::READING_FOR_PREDICATE;
}
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 26df6e69c2e..84d1fe242da 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -778,17 +778,7 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
// set column name to apply access paths.
(*iter)->set_column_name(tablet_column.name());
- auto st = (*iter)->set_access_paths(all_access_paths, predicate_access_paths);
- if (!st.ok()) {
- std::stringstream ss;
- all_access_paths.printTo(ss);
- std::stringstream ss2;
- predicate_access_paths.printTo(ss2);
- LOG(WARNING) << "set access paths failed, column_uid=" << unique_id
- << ", all_access_paths=" << ss.str()
- << ", predicate_access_paths=" << ss2.str();
- return st;
- }
+ RETURN_IF_ERROR((*iter)->set_access_paths(all_access_paths, predicate_access_paths));
(*iter)->remove_pruned_sub_iterators();
}
}
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 7ff16f6f5e2..c66ccd80b1d 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -53,6 +53,8 @@ class ObjectPool;
class PTupleDescriptor;
class PSlotDescriptor;
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
class SlotDescriptor {
public:
MOCK_DEFINE(virtual ~SlotDescriptor() = default;)
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index ab4e69e2294..b8cd37f4a09 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -542,18 +542,18 @@ Status OlapScanner::_init_return_columns() {
}
const auto& column = tablet_schema->column(index);
- if (!slot->all_access_paths().name_access_paths.empty()) {
+ if (!slot->all_access_paths().empty()) {
_tablet_reader_params.all_access_paths.insert(
{column.unique_id(), slot->all_access_paths()});
}
- if (!slot->predicate_access_paths().name_access_paths.empty()) {
+ if (!slot->predicate_access_paths().empty()) {
_tablet_reader_params.predicate_access_paths.insert(
{column.unique_id(), slot->predicate_access_paths()});
}
if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT &&
- !slot->all_access_paths().name_access_paths.empty()) {
+ !slot->all_access_paths().empty()) {
tablet_schema->add_pruned_columns_data_type(column.unique_id(), slot->type());
}
diff --git a/be/test/olap/rowset/segment_v2/column_reader_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
index 495a0dcaf4e..98478df7016 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
@@ -72,10 +72,10 @@ TEST_F(ColumnReaderTest, StructAccessPaths) {
ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NORMAL_READING);
TColumnAccessPaths all_access_paths;
- all_access_paths.name_access_paths.emplace_back();
+ all_access_paths.emplace_back();
TColumnAccessPaths predicate_access_paths;
- predicate_access_paths.name_access_paths.emplace_back();
+ predicate_access_paths.emplace_back();
st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
// empty paths leads to error
@@ -83,9 +83,9 @@ TEST_F(ColumnReaderTest, StructAccessPaths) {
// Only reading sub_col_1
// sub_col_2 should be set to SKIP_READING
- all_access_paths.name_access_paths[0].path = {"self", "sub_col_1"};
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
- predicate_access_paths.name_access_paths[0].path = {"self", "sub_col_1"};
+ predicate_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
// invalid name leads to error
@@ -103,7 +103,7 @@ TEST_F(ColumnReaderTest, StructAccessPaths) {
ColumnIterator::ReadingFlag::SKIP_READING);
// Reading all sub columns
- all_access_paths.name_access_paths[0].path = {"self"};
+ all_access_paths[0].data_access_path.path = {"self"};
iterator = create_struct_iterator();
iterator->set_column_name("self");
st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
@@ -189,12 +189,12 @@ TEST_F(ColumnReaderTest, MultiAccessPaths) {
array_iterator->set_column_name("sub_col_2");
auto iterator = create_struct_iterator2(std::move(array_iterator));
TColumnAccessPaths all_access_paths;
- all_access_paths.name_access_paths.emplace_back();
+ all_access_paths.emplace_back();
// all access paths:
// self.sub_col_2.*.KEYS
// predicates paths empty
- all_access_paths.name_access_paths[0].path = {"self", "sub_col_2", "*", "KEYS"};
+ all_access_paths[0].data_access_path.path = {"self", "sub_col_2", "*", "KEYS"};
TColumnAccessPaths predicate_access_paths;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org