You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mr...@apache.org on 2025/10/27 06:32:57 UTC

(doris) branch nested_column_prune updated: Support new thrift protocol(data/meta)

This is an automated email from the ASF dual-hosted git repository.

mrhhsg pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/nested_column_prune by this push:
     new 79b18201e23 Support new thrift protocol(data/meta)
79b18201e23 is described below

commit 79b18201e23e94c48ffb888749df296f82353e1d
Author: Hu Shenggang <hu...@selectdb.com>
AuthorDate: Mon Oct 27 14:32:01 2025 +0800

    Support new thrift protocol(data/meta)
---
 be/src/olap/iterators.h                            |   2 +
 be/src/olap/rowset/rowset_reader_context.h         |   2 +
 be/src/olap/rowset/segment_v2/column_reader.cpp    | 119 ++++++++++-----------
 be/src/olap/rowset/segment_v2/column_reader.h      |   4 +-
 be/src/olap/rowset/segment_v2/segment.cpp          |  12 +--
 be/src/runtime/descriptors.h                       |   2 +
 be/src/vec/exec/scan/olap_scanner.cpp              |   6 +-
 .../olap/rowset/segment_v2/column_reader_test.cpp  |  14 +--
 8 files changed, 79 insertions(+), 82 deletions(-)

diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index f84a25c3720..bb14f7fc2b4 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -47,6 +47,8 @@ namespace segment_v2 {
 struct SubstreamIterator;
 }
 
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
 class StorageReadOptions {
 public:
     struct KeyRange {
diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h
index 7caec564af9..f3b4f9eb597 100644
--- a/be/src/olap/rowset/rowset_reader_context.h
+++ b/be/src/olap/rowset/rowset_reader_context.h
@@ -18,6 +18,8 @@
 #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
 #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H
 
+#include <vector>
+
 #include "io/io_common.h"
 #include "olap/column_predicate.h"
 #include "olap/olap_common.h"
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index fff3c6c3f42..cd00f58954a 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -929,26 +929,25 @@ Status ColumnReader::new_struct_iterator(ColumnIteratorUPtr* iterator,
 Result<TColumnAccessPaths> ColumnIterator::_get_sub_access_paths(
         const TColumnAccessPaths& access_paths) {
     TColumnAccessPaths sub_access_paths = access_paths;
-    for (auto it = sub_access_paths.name_access_paths.begin();
-         it != sub_access_paths.name_access_paths.end();) {
-        TColumnNameAccessPath& name_path = *it;
-        if (name_path.path.empty()) {
+    for (auto it = sub_access_paths.begin(); it != sub_access_paths.end();) {
+        TColumnAccessPath& name_path = *it;
+        if (name_path.data_access_path.path.empty()) {
             return ResultError(
                     Status::InternalError("Invalid access path for struct column: path is empty"));
         }
 
-        if (name_path.path[0] != _column_name) {
+        if (name_path.data_access_path.path[0] != _column_name) {
             return ResultError(Status::InternalError(
                     R"(Invalid access path for column: expected name "{}", got "{}")", _column_name,
-                    name_path.path[0]));
+                    name_path.data_access_path.path[0]));
         }
 
-        name_path.path.erase(name_path.path.begin());
-        if (!name_path.path.empty()) {
+        name_path.data_access_path.path.erase(name_path.data_access_path.path.begin());
+        if (!name_path.data_access_path.path.empty()) {
             ++it;
         } else {
             set_need_to_read();
-            it = sub_access_paths.name_access_paths.erase(it);
+            it = sub_access_paths.erase(it);
         }
     }
     return sub_access_paths;
@@ -1091,11 +1090,11 @@ void MapFileColumnIterator::remove_pruned_sub_iterators() {
 
 Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
                                                const TColumnAccessPaths& predicate_access_paths) {
-    if (all_access_paths.name_access_paths.empty()) {
+    if (all_access_paths.empty()) {
         return Status::OK();
     }
 
-    if (!predicate_access_paths.name_access_paths.empty()) {
+    if (!predicate_access_paths.empty()) {
         set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
         LOG(INFO) << "Map column iterator set sub-column " << _column_name
                   << " to READING_FOR_PREDICATE";
@@ -1104,7 +1103,7 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc
     auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
     auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
 
-    if (sub_all_access_paths.name_access_paths.empty()) {
+    if (sub_all_access_paths.empty()) {
         return Status::OK();
     }
 
@@ -1113,35 +1112,35 @@ Status MapFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_acc
     TColumnAccessPaths key_predicate_access_paths;
     TColumnAccessPaths val_predicate_access_paths;
 
-    for (auto paths : sub_all_access_paths.name_access_paths) {
-        if (paths.path[0] == "*") {
-            paths.path[0] = _key_iterator->column_name();
-            key_all_access_paths.name_access_paths.push_back(paths);
-            paths.path[0] = _val_iterator->column_name();
-            val_all_access_paths.name_access_paths.push_back(paths);
-        } else if (paths.path[0] == "KEYS") {
-            paths.path[0] = _key_iterator->column_name();
-            key_all_access_paths.name_access_paths.push_back(paths);
-        } else if (paths.path[0] == "VALUES") {
-            paths.path[0] = _val_iterator->column_name();
-            val_all_access_paths.name_access_paths.push_back(paths);
+    for (auto paths : sub_all_access_paths) {
+        if (paths.data_access_path.path[0] == "*") {
+            paths.data_access_path.path[0] = _key_iterator->column_name();
+            key_all_access_paths.emplace_back(paths);
+            paths.data_access_path.path[0] = _val_iterator->column_name();
+            val_all_access_paths.emplace_back(paths);
+        } else if (paths.data_access_path.path[0] == "KEYS") {
+            paths.data_access_path.path[0] = _key_iterator->column_name();
+            key_all_access_paths.emplace_back(paths);
+        } else if (paths.data_access_path.path[0] == "VALUES") {
+            paths.data_access_path.path[0] = _val_iterator->column_name();
+            val_all_access_paths.emplace_back(paths);
         }
     }
-    const auto need_read_keys = !key_all_access_paths.name_access_paths.empty();
-    const auto need_read_values = !val_all_access_paths.name_access_paths.empty();
-
-    for (auto paths : sub_predicate_access_paths.name_access_paths) {
-        if (paths.path[0] == "*") {
-            paths.path[0] = _key_iterator->column_name();
-            key_predicate_access_paths.name_access_paths.push_back(paths);
-            paths.path[0] = _val_iterator->column_name();
-            val_predicate_access_paths.name_access_paths.push_back(paths);
-        } else if (paths.path[0] == "KEYS") {
-            paths.path[0] = _key_iterator->column_name();
-            key_predicate_access_paths.name_access_paths.push_back(paths);
-        } else if (paths.path[0] == "VALUES") {
-            paths.path[0] = _val_iterator->column_name();
-            val_predicate_access_paths.name_access_paths.push_back(paths);
+    const auto need_read_keys = !key_all_access_paths.empty();
+    const auto need_read_values = !val_all_access_paths.empty();
+
+    for (auto paths : sub_predicate_access_paths) {
+        if (paths.data_access_path.path[0] == "*") {
+            paths.data_access_path.path[0] = _key_iterator->column_name();
+            key_predicate_access_paths.emplace_back(paths);
+            paths.data_access_path.path[0] = _val_iterator->column_name();
+            val_predicate_access_paths.emplace_back(paths);
+        } else if (paths.data_access_path.path[0] == "KEYS") {
+            paths.data_access_path.path[0] = _key_iterator->column_name();
+            key_predicate_access_paths.emplace_back(paths);
+        } else if (paths.data_access_path.path[0] == "VALUES") {
+            paths.data_access_path.path[0] = _val_iterator->column_name();
+            val_predicate_access_paths.emplace_back(paths);
         }
     }
 
@@ -1290,11 +1289,11 @@ void StructFileColumnIterator::remove_pruned_sub_iterators() {
 Status StructFileColumnIterator::set_access_paths(
         const TColumnAccessPaths& all_access_paths,
         const TColumnAccessPaths& predicate_access_paths) {
-    if (all_access_paths.name_access_paths.empty()) {
+    if (all_access_paths.empty()) {
         return Status::OK();
     }
 
-    if (!predicate_access_paths.name_access_paths.empty()) {
+    if (!predicate_access_paths.empty()) {
         set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
         LOG(INFO) << "Struct column iterator set sub-column " << _column_name
                   << " to READING_FOR_PREDICATE";
@@ -1302,20 +1301,20 @@ Status StructFileColumnIterator::set_access_paths(
     auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
     auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
 
-    const auto no_sub_column_to_skip = sub_all_access_paths.name_access_paths.empty();
-    const auto no_predicate_sub_column = sub_predicate_access_paths.name_access_paths.empty();
+    const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+    const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
 
     for (auto& sub_iterator : _sub_column_iterators) {
         const auto name = sub_iterator->column_name();
         bool need_to_read = no_sub_column_to_skip;
         TColumnAccessPaths sub_all_access_paths_of_this;
         if (!need_to_read) {
-            for (const auto& paths : sub_all_access_paths.name_access_paths) {
-                if (paths.path[0] == name) {
-                    sub_all_access_paths_of_this.name_access_paths.push_back(paths);
+            for (const auto& paths : sub_all_access_paths) {
+                if (paths.data_access_path.path[0] == name) {
+                    sub_all_access_paths_of_this.emplace_back(paths);
                 }
             }
-            need_to_read = !sub_all_access_paths_of_this.name_access_paths.empty();
+            need_to_read = !sub_all_access_paths_of_this.empty();
         }
 
         if (!need_to_read) {
@@ -1330,9 +1329,9 @@ Status StructFileColumnIterator::set_access_paths(
         TColumnAccessPaths sub_predicate_access_paths_of_this;
 
         if (!no_predicate_sub_column) {
-            for (const auto& paths : sub_predicate_access_paths.name_access_paths) {
-                if (paths.path[0] == name) {
-                    sub_predicate_access_paths_of_this.name_access_paths.push_back(paths);
+            for (const auto& paths : sub_predicate_access_paths) {
+                if (paths.data_access_path.path[0] == name) {
+                    sub_predicate_access_paths_of_this.emplace_back(paths);
                 }
             }
         }
@@ -1531,11 +1530,11 @@ void ArrayFileColumnIterator::remove_pruned_sub_iterators() {
 
 Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_access_paths,
                                                  const TColumnAccessPaths& predicate_access_paths) {
-    if (all_access_paths.name_access_paths.empty()) {
+    if (all_access_paths.empty()) {
         return Status::OK();
     }
 
-    if (!predicate_access_paths.name_access_paths.empty()) {
+    if (!predicate_access_paths.empty()) {
         set_reading_flag(ReadingFlag::READING_FOR_PREDICATE);
         LOG(INFO) << "Array column iterator set sub-column " << _column_name
                   << " to READING_FOR_PREDICATE";
@@ -1544,21 +1543,21 @@ Status ArrayFileColumnIterator::set_access_paths(const TColumnAccessPaths& all_a
     auto sub_all_access_paths = DORIS_TRY(_get_sub_access_paths(all_access_paths));
     auto sub_predicate_access_paths = DORIS_TRY(_get_sub_access_paths(predicate_access_paths));
 
-    const auto no_sub_column_to_skip = sub_all_access_paths.name_access_paths.empty();
-    const auto no_predicate_sub_column = sub_predicate_access_paths.name_access_paths.empty();
+    const auto no_sub_column_to_skip = sub_all_access_paths.empty();
+    const auto no_predicate_sub_column = sub_predicate_access_paths.empty();
 
     if (!no_sub_column_to_skip) {
-        for (auto& path : sub_all_access_paths.name_access_paths) {
-            if (path.path[0] == "*") {
-                path.path[0] = _item_iterator->column_name();
+        for (auto& path : sub_all_access_paths) {
+            if (path.data_access_path.path[0] == "*") {
+                path.data_access_path.path[0] = _item_iterator->column_name();
             }
         }
     }
 
     if (!no_predicate_sub_column) {
-        for (auto& path : sub_predicate_access_paths.name_access_paths) {
-            if (path.path[0] == "*") {
-                path.path[0] = _item_iterator->column_name();
+        for (auto& path : sub_predicate_access_paths) {
+            if (path.data_access_path.path[0] == "*") {
+                path.data_access_path.path[0] = _item_iterator->column_name();
             }
         }
     }
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 0abd06f937e..fddbc94c213 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -70,6 +70,8 @@ class FileReader;
 struct Slice;
 struct StringRef;
 
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
 namespace segment_v2 {
 
 class EncodingInfo;
@@ -368,7 +370,7 @@ public:
 
     virtual Status set_access_paths(const TColumnAccessPaths& all_access_paths,
                                     const TColumnAccessPaths& predicate_access_paths) {
-        if (!predicate_access_paths.name_access_paths.empty()) {
+        if (!predicate_access_paths.empty()) {
             _reading_flag = ReadingFlag::READING_FOR_PREDICATE;
         }
         return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 26df6e69c2e..84d1fe242da 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -778,17 +778,7 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
 
             // set column name to apply access paths.
             (*iter)->set_column_name(tablet_column.name());
-            auto st = (*iter)->set_access_paths(all_access_paths, predicate_access_paths);
-            if (!st.ok()) {
-                std::stringstream ss;
-                all_access_paths.printTo(ss);
-                std::stringstream ss2;
-                predicate_access_paths.printTo(ss2);
-                LOG(WARNING) << "set access paths failed, column_uid=" << unique_id
-                             << ", all_access_paths=" << ss.str()
-                             << ", predicate_access_paths=" << ss2.str();
-                return st;
-            }
+            RETURN_IF_ERROR((*iter)->set_access_paths(all_access_paths, predicate_access_paths));
             (*iter)->remove_pruned_sub_iterators();
         }
     }
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 7ff16f6f5e2..c66ccd80b1d 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -53,6 +53,8 @@ class ObjectPool;
 class PTupleDescriptor;
 class PSlotDescriptor;
 
+using TColumnAccessPaths = std::vector<TColumnAccessPath>;
+
 class SlotDescriptor {
 public:
     MOCK_DEFINE(virtual ~SlotDescriptor() = default;)
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index ab4e69e2294..b8cd37f4a09 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -542,18 +542,18 @@ Status OlapScanner::_init_return_columns() {
         }
 
         const auto& column = tablet_schema->column(index);
-        if (!slot->all_access_paths().name_access_paths.empty()) {
+        if (!slot->all_access_paths().empty()) {
             _tablet_reader_params.all_access_paths.insert(
                     {column.unique_id(), slot->all_access_paths()});
         }
 
-        if (!slot->predicate_access_paths().name_access_paths.empty()) {
+        if (!slot->predicate_access_paths().empty()) {
             _tablet_reader_params.predicate_access_paths.insert(
                     {column.unique_id(), slot->predicate_access_paths()});
         }
 
         if (slot->type()->get_primitive_type() == PrimitiveType::TYPE_STRUCT &&
-            !slot->all_access_paths().name_access_paths.empty()) {
+            !slot->all_access_paths().empty()) {
             tablet_schema->add_pruned_columns_data_type(column.unique_id(), slot->type());
         }
 
diff --git a/be/test/olap/rowset/segment_v2/column_reader_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
index 495a0dcaf4e..98478df7016 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_test.cpp
@@ -72,10 +72,10 @@ TEST_F(ColumnReaderTest, StructAccessPaths) {
     ASSERT_EQ(iterator->_reading_flag, ColumnIterator::ReadingFlag::NORMAL_READING);
 
     TColumnAccessPaths all_access_paths;
-    all_access_paths.name_access_paths.emplace_back();
+    all_access_paths.emplace_back();
 
     TColumnAccessPaths predicate_access_paths;
-    predicate_access_paths.name_access_paths.emplace_back();
+    predicate_access_paths.emplace_back();
 
     st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
     // empty paths leads to error
@@ -83,9 +83,9 @@ TEST_F(ColumnReaderTest, StructAccessPaths) {
 
     // Only reading sub_col_1
     // sub_col_2 should be set to SKIP_READING
-    all_access_paths.name_access_paths[0].path = {"self", "sub_col_1"};
+    all_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
 
-    predicate_access_paths.name_access_paths[0].path = {"self", "sub_col_1"};
+    predicate_access_paths[0].data_access_path.path = {"self", "sub_col_1"};
 
     st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
     // invalid name leads to error
@@ -103,7 +103,7 @@ TEST_F(ColumnReaderTest, StructAccessPaths) {
               ColumnIterator::ReadingFlag::SKIP_READING);
 
     // Reading all sub columns
-    all_access_paths.name_access_paths[0].path = {"self"};
+    all_access_paths[0].data_access_path.path = {"self"};
     iterator = create_struct_iterator();
     iterator->set_column_name("self");
     st = iterator->set_access_paths(all_access_paths, predicate_access_paths);
@@ -189,12 +189,12 @@ TEST_F(ColumnReaderTest, MultiAccessPaths) {
     array_iterator->set_column_name("sub_col_2");
     auto iterator = create_struct_iterator2(std::move(array_iterator));
     TColumnAccessPaths all_access_paths;
-    all_access_paths.name_access_paths.emplace_back();
+    all_access_paths.emplace_back();
 
     // all access paths:
     // self.sub_col_2.*.KEYS
     // predicates paths empty
-    all_access_paths.name_access_paths[0].path = {"self", "sub_col_2", "*", "KEYS"};
+    all_access_paths[0].data_access_path.path = {"self", "sub_col_2", "*", "KEYS"};
 
     TColumnAccessPaths predicate_access_paths;
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org