You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ka...@apache.org on 2025/10/24 04:56:44 UTC
(doris) branch nested_column_prune_external_table_no_late_mat created (now c51d31547f6)
This is an automated email from the ASF dual-hosted git repository.
kakachen pushed a change to branch nested_column_prune_external_table_no_late_mat
in repository https://gitbox.apache.org/repos/asf/doris.git
at c51d31547f6 [Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
This branch includes the following new commits:
new c51d31547f6 [Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/01: [Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kakachen pushed a commit to branch nested_column_prune_external_table_no_late_mat
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c51d31547f67695689ce5b3e09918ce86e73f344
Author: kakachen <ch...@selectdb.com>
AuthorDate: Thu Oct 23 09:54:32 2025 +0800
[Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
---
.gitmodules | 2 +-
be/src/runtime/descriptors.cpp | 10 +-
be/src/runtime/descriptors.h | 16 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 311 ++++-
be/src/vec/exec/format/orc/vorc_reader.h | 24 +-
be/src/vec/exec/format/parquet/schema_desc.cpp | 87 +-
be/src/vec/exec/format/parquet/schema_desc.h | 30 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 81 +-
.../exec/format/parquet/vparquet_column_reader.h | 91 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 19 +-
.../exec/format/parquet/vparquet_group_reader.h | 8 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 21 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 5 +-
.../table/hive/hive_orc_nested_column_utils.cpp | 151 +++
.../table/hive/hive_orc_nested_column_utils.h | 43 +
.../hive/hive_parquet_nested_column_utils.cpp | 173 +++
.../table/hive/hive_parquet_nested_column_utils.h | 41 +
be/src/vec/exec/format/table/hive_reader.cpp | 435 ++++++-
be/src/vec/exec/format/table/hive_reader.h | 14 +
.../iceberg/iceberg_orc_nested_column_utils.cpp | 149 +++
.../iceberg/iceberg_orc_nested_column_utils.h | 46 +
.../iceberg_parquet_nested_column_utils.cpp | 173 +++
.../iceberg/iceberg_parquet_nested_column_utils.h | 42 +
be/src/vec/exec/format/table/iceberg_reader.cpp | 235 +++-
be/src/vec/exec/format/table/iceberg_reader.h | 7 +
.../vec/exec/format/table/table_format_reader.cpp | 21 +-
be/src/vec/exec/format/table/table_format_reader.h | 11 +
be/src/vec/exec/scan/file_scanner.cpp | 6 +-
be/src/vec/exec/scan/file_scanner.h | 2 +-
be/src/vec/exec/scan/olap_scanner.cpp | 8 +-
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2277 bytes
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2334 bytes
...2c4c8-e0bd-4db6-b550-6323977446a4.metadata.json | 145 +++
...66011-3129-4bfb-9b4b-c1510c566d73.metadata.json | 176 +++
.../6a876742-85c2-4f84-ae24-292482fa9288-m0.avro | Bin 0 -> 8175 bytes
...212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro | Bin 0 -> 4264 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4068 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4020 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4043 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4082 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4067 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4047 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4077 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4147 bytes
...38ed9-31af-46c8-9274-506c7f2709c9.metadata.json | 145 +++
...4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json | 176 +++
.../51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro | Bin 0 -> 8783 bytes
...552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro | Bin 0 -> 4264 bytes
...a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc | Bin 0 -> 7181 bytes
...20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json | 548 +++++++++
...ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json | 579 ++++++++++
.../d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro | 1 +
...228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro | Bin 0 -> 4261 bytes
...35f-d581-40e4-8d56-a929aded99e4-0-00001.parquet | Bin 0 -> 21586 bytes
...ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json | 548 +++++++++
...56265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json | 579 ++++++++++
.../bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro | 5 +
...002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro | Bin 0 -> 4262 bytes
.../test_data/nested_user_profiles_orc/_SUCCESS | 0
...f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc | Bin 0 -> 8396 bytes
.../nested_user_profiles_parquet/_SUCCESS | 0
...1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet | Bin 0 -> 21157 bytes
.../exec/format/parquet/parquet_thrift_test.cpp | 12 +-
.../hive/hive_reader_create_column_ids_test.cpp | 1183 ++++++++++++++++++++
.../exec/format/table/hive/hive_reader_test.cpp | 733 ++++++++++++
.../iceberg_reader_create_column_ids_test.cpp | 1181 +++++++++++++++++++
.../format/table/iceberg/iceberg_reader_test.cpp | 729 ++++++++++++
contrib/apache-orc | 2 +-
.../expressions/functions/scalar/ElementAt.java | 3 +-
.../trees/plans/logical/LogicalFileScan.java | 27 +-
.../trees/plans/logical/LogicalHudiScan.java | 2 +-
.../trees/plans/logical/LogicalOlapScan.java | 3 +-
72 files changed, 8909 insertions(+), 130 deletions(-)
diff --git a/.gitmodules b/.gitmodules
index 54c1a8a3636..63e4b14f70d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -20,7 +20,7 @@
[submodule "contrib/apache-orc"]
path = contrib/apache-orc
url = https://github.com/apache/doris-thirdparty.git
- branch = orc
+ branch = cq_nested_column_prune_external_table
[submodule "doris-faiss"]
path = contrib/faiss
url = https://github.com/apache/doris-thirdparty.git
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index a17f0926752..c410db985eb 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -63,11 +63,11 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc)
_is_materialized(tdesc.isMaterialized && tdesc.need_materialize),
_is_key(tdesc.is_key),
_column_paths(tdesc.column_paths),
- _all_access_paths(tdesc.__isset.all_access_paths ? tdesc.all_access_paths
- : TColumnAccessPaths {}),
- _predicate_access_paths(tdesc.__isset.predicate_access_paths
- ? tdesc.predicate_access_paths
- : TColumnAccessPaths {}),
+ _all_column_access_paths(tdesc.__isset.all_access_paths ? tdesc.all_access_paths
+ : TColumnAccessPaths {}),
+ _predicate_column_access_paths(tdesc.__isset.predicate_access_paths
+ ? tdesc.predicate_access_paths
+ : TColumnAccessPaths {}),
_is_auto_increment(tdesc.__isset.is_auto_increment ? tdesc.is_auto_increment : false),
_col_default_value(tdesc.__isset.col_default_value ? tdesc.col_default_value : "") {
if (tdesc.__isset.virtual_column_expr) {
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index 7ff16f6f5e2..5d114ab729b 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -82,8 +82,10 @@ public:
bool is_key() const { return _is_key; }
const std::vector<std::string>& column_paths() const { return _column_paths; };
- const TColumnAccessPaths& all_access_paths() const { return _all_access_paths; }
- const TColumnAccessPaths& predicate_access_paths() const { return _predicate_access_paths; }
+ const TColumnAccessPaths& all_column_access_paths() const { return _all_column_access_paths; }
+ const TColumnAccessPaths& predicate_column_access_paths() const {
+ return _predicate_column_access_paths;
+ }
bool is_auto_increment() const { return _is_auto_increment; }
@@ -98,6 +100,10 @@ public:
return virtual_column_expr;
}
+ void set_is_predicate(bool is_predicate) { _is_predicate = is_predicate; }
+
+ bool is_predicate() const { return _is_predicate; }
+
private:
friend class DescriptorTbl;
friend class TupleDescriptor;
@@ -131,14 +137,16 @@ private:
const bool _is_key;
const std::vector<std::string> _column_paths;
- const TColumnAccessPaths _all_access_paths;
- const TColumnAccessPaths _predicate_access_paths;
+ const TColumnAccessPaths _all_column_access_paths;
+ const TColumnAccessPaths _predicate_column_access_paths;
const bool _is_auto_increment;
const std::string _col_default_value;
std::shared_ptr<doris::TExpr> virtual_column_expr = nullptr;
+ bool _is_predicate = false;
+
SlotDescriptor(const TSlotDescriptor& tdesc);
SlotDescriptor(const PSlotDescriptor& pdesc);
MOCK_DEFINE(SlotDescriptor();)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index bd996f2189b..203e917b271 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -26,6 +26,7 @@
#include <algorithm>
#include <cctype>
+#include <list>
#include "vec/exprs/vdirect_in_predicate.h"
#include "vec/exprs/vexpr.h"
@@ -355,8 +356,11 @@ Status OrcReader::init_reader(
const RowDescriptor* row_descriptor,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
- std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr,
+ const std::set<uint64_t>& column_ids, const std::set<uint64_t>& filter_column_ids) {
_table_column_names = column_names;
+ _column_ids = column_ids;
+ _filter_column_ids = filter_column_ids;
_colname_to_value_range = colname_to_value_range;
_lazy_read_ctx.conjuncts = conjuncts;
_is_acid = is_acid;
@@ -401,6 +405,32 @@ Status OrcReader::get_parsed_schema(std::vector<std::string>* col_names,
Status OrcReader::_init_read_columns() {
SCOPED_RAW_TIMER(&_statistics.init_column_time);
const auto& root_type = _reader->getType();
+
+ // Save file root type for later use in type conversion
+ _file_root_type = &root_type;
+
+ // Build column ID to file type mapping for all columns in file
+ _column_id_to_file_type.clear();
+ std::function<void(const orc::Type*, int)> build_column_id_map = [&](const orc::Type* type,
+ int depth) {
+ if (type == nullptr) return;
+ uint64_t col_id = type->getColumnId();
+ _column_id_to_file_type[col_id] = type;
+
+ std::string indent(depth * 2, ' ');
+ VLOG_DEBUG << indent << "[OrcReader] Mapping column_id=" << col_id
+ << ", kind=" << static_cast<int>(type->getKind())
+ << ", subtype_count=" << type->getSubtypeCount();
+
+ for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
+ build_column_id_map(type->getSubtype(i), depth + 1);
+ }
+ };
+
+ VLOG_DEBUG << "[OrcReader] Building column ID to file type mapping...";
+ build_column_id_map(&root_type, 0);
+ VLOG_DEBUG << "[OrcReader] Total mapped columns: " << _column_id_to_file_type.size();
+
if (_is_acid) {
for (uint64_t i = 0; i < root_type.getSubtypeCount(); ++i) {
if (root_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) {
@@ -650,7 +680,7 @@ std::pair<bool, orc::Literal> OrcReader::_make_orc_literal(const VSlotRef* slot_
auto* slot = _tuple_descriptor->slots()[slot_ref->column_id()];
auto slot_type = slot->type();
auto primitive_type = slot_type->get_primitive_type();
- auto src_type = OrcReader::convert_to_doris_type(orc_type)->get_primitive_type();
+ auto src_type = convert_to_doris_type(orc_type)->get_primitive_type();
// should not down predicate for string type change from other type
if (src_type != primitive_type && !is_string_type(src_type) && is_string_type(primitive_type)) {
LOG(WARNING) << "Unsupported Push Down Schema Changed Column " << primitive_type << " to "
@@ -1158,7 +1188,13 @@ Status OrcReader::set_fill_columns(
try {
_row_reader_options.range(_range_start_offset, _range_size);
_row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" : _ctz);
- _row_reader_options.include(_read_file_cols);
+ if (!_column_ids.empty()) {
+ // Convert std::set to std::list for includeTypes
+ std::list<uint64_t> column_ids_list(_column_ids.begin(), _column_ids.end());
+ _row_reader_options.includeTypes(column_ids_list);
+ } else {
+ _row_reader_options.include(_read_file_cols);
+ }
_row_reader_options.setEnableLazyDecoding(true);
//orc reader should not use the tiny stripe optimization when reading by row id.
@@ -1246,6 +1282,57 @@ Status OrcReader::set_fill_columns(
}
}
+ // Helper function to recursively print ORC type structure
+ std::function<void(const orc::Type*, int, const std::string&)> print_orc_type_tree =
+ [&](const orc::Type* type, int indent, const std::string& name) {
+ if (type) {
+ std::string indent_str(indent * 2, ' ');
+ VLOG_DEBUG << indent_str << "- " << name
+ << " (type_id=" << type->getColumnId()
+ << ", kind=" << static_cast<int>(type->getKind()) << ")";
+ }
+
+ if (type->getKind() == orc::TypeKind::STRUCT) {
+ for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
+ std::string field_name = type->getFieldName(i);
+ print_orc_type_tree(type->getSubtype(i), indent + 1, field_name);
+ }
+ } else if (type->getKind() == orc::TypeKind::LIST) {
+ print_orc_type_tree(type->getSubtype(0), indent + 1, "element");
+ } else if (type->getKind() == orc::TypeKind::MAP) {
+ print_orc_type_tree(type->getSubtype(0), indent + 1, "key");
+ print_orc_type_tree(type->getSubtype(1), indent + 1, "value");
+ } else {
+ // Primitive type, no children
+ }
+ };
+
+ // VLOG_DEBUG << "\n[OrcReader] ========== ORC Type Structure ==========";
+ // print_orc_type_tree(&selected_type, 0, "root");
+ // VLOG_DEBUG << "[OrcReader] ==========================================\n";
+
+ _type_map.clear();
+ if (_is_acid) {
+ for (uint64_t i = 0; i < selected_type.getSubtypeCount(); ++i) {
+ if (selected_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) {
+ auto row_orc_type = selected_type.getSubtype(i);
+ for (uint64_t j = 0; j < row_orc_type->getSubtypeCount(); j++) {
+ std::string field_name =
+ TransactionalHive::ROW + "." + row_orc_type->getFieldName(j);
+ _type_map.emplace(field_name, row_orc_type->getSubtype(j));
+ }
+ } else {
+ std::string field_name = selected_type.getFieldName(i);
+ _type_map.emplace(field_name, selected_type.getSubtype(i));
+ }
+ }
+ } else {
+ for (int i = 0; i < selected_type.getSubtypeCount(); ++i) {
+ std::string field_name = selected_type.getFieldName(i);
+ _type_map.emplace(field_name, selected_type.getSubtype(i));
+ }
+ }
+
_remaining_rows = _row_reader->getNumberOfRows();
} catch (std::exception& e) {
@@ -1391,6 +1478,14 @@ void OrcReader::_init_file_description() {
}
DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
+ // Critical: check for nullptr BEFORE accessing any methods
+ if (orc_type == nullptr) {
+ LOG(WARNING) << "[OrcReader] ERROR: convert_to_doris_type called with nullptr orc_type! "
+ "Falling back to STRING";
+ // Return a safe default type instead of crashing
+ return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true);
+ }
+
switch (orc_type->getKind()) {
case orc::TypeKind::BOOLEAN:
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BOOLEAN, true);
@@ -1437,6 +1532,74 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
std::make_shared<DataTypeArray>(convert_to_doris_type(orc_type->getSubtype(0))));
}
case orc::TypeKind::MAP: {
+ // Handle incomplete MAP type due to column pruning
+ // If MAP doesn't have both key and value subtypes, try to find the complete type from file
+ if (orc_type->getSubtypeCount() < 2 || orc_type->getSubtype(0) == nullptr ||
+ orc_type->getSubtype(1) == nullptr) {
+ // Try to find the complete type from _column_id_to_file_type
+ uint64_t column_id = orc_type->getColumnId();
+ VLOG(1) << "[OrcReader] Detected incomplete MAP type: column_id=" << column_id
+ << ", subtype_count=" << orc_type->getSubtypeCount() << ", subtype(0)="
+ << (orc_type->getSubtypeCount() > 0 && orc_type->getSubtype(0) != nullptr
+ ? "not null"
+ : "null")
+ << ", subtype(1)="
+ << (orc_type->getSubtypeCount() > 1 && orc_type->getSubtype(1) != nullptr
+ ? "not null"
+ : "null");
+
+ auto it = _column_id_to_file_type.find(column_id);
+ if (it != _column_id_to_file_type.end() && it->second != nullptr) {
+ const orc::Type* complete_type = it->second;
+ VLOG(1) << "[OrcReader] Found complete type in mapping: column_id=" << column_id
+ << ", complete_type_kind=" << static_cast<int>(complete_type->getKind())
+ << ", complete_subtype_count=" << complete_type->getSubtypeCount();
+
+ // Print subtypes information
+ for (uint64_t i = 0; i < complete_type->getSubtypeCount(); ++i) {
+ const orc::Type* subtype = complete_type->getSubtype(i);
+ VLOG_DEBUG << "[OrcReader] complete_type->subtype[" << i << "]: "
+ << (subtype != nullptr
+ ? ("kind=" +
+ std::to_string(static_cast<int>(subtype->getKind())) +
+ ", column_id=" +
+ std::to_string(subtype->getColumnId()))
+ : "nullptr");
+ }
+
+ if (complete_type->getKind() == orc::TypeKind::MAP &&
+ complete_type->getSubtypeCount() == 2) {
+ VLOG(1) << "[OrcReader] Using complete MAP type from file schema for column_id="
+ << column_id;
+
+ // Get subtypes with extra validation
+ const orc::Type* key_type = complete_type->getSubtype(0);
+ const orc::Type* value_type = complete_type->getSubtype(1);
+
+ VLOG_DEBUG << "[OrcReader] About to convert key_type: "
+ << (key_type != nullptr ? "not null" : "NULL");
+ VLOG_DEBUG << "[OrcReader] About to convert value_type: "
+ << (value_type != nullptr ? "not null" : "NULL");
+
+ // Use the complete type from file - with null checks
+ DataTypePtr key_doris_type = convert_to_doris_type(key_type);
+ VLOG_DEBUG << "[OrcReader] Successfully converted key_type";
+
+ DataTypePtr value_doris_type = convert_to_doris_type(value_type);
+ VLOG_DEBUG << "[OrcReader] Successfully converted value_type";
+
+ return make_nullable(
+ std::make_shared<DataTypeMap>(key_doris_type, value_doris_type));
+ } else {
+ LOG(WARNING) << "[OrcReader] Warning: Complete type is not a valid MAP or has "
+ "wrong subtype count";
+ }
+ } else {
+ LOG(WARNING) << "[OrcReader] Warning: Could not find complete type in mapping for "
+ "column_id="
+ << column_id << ", mapping_size=" << _column_id_to_file_type.size();
+ }
+ }
return make_nullable(
std::make_shared<DataTypeMap>(convert_to_doris_type(orc_type->getSubtype(0)),
convert_to_doris_type(orc_type->getSubtype(1))));
@@ -1682,6 +1845,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const orc::Type* orc_column_type,
const orc::ColumnVectorBatch* cvb, size_t num_values) {
auto logical_type = data_type->get_primitive_type();
+
switch (logical_type) {
#define DISPATCH(FlatType, CppType, OrcColumnType) \
case FlatType: \
@@ -1752,21 +1916,99 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const DataTypePtr& doris_value_type =
reinterpret_cast<const DataTypeMap*>(remove_nullable(data_type).get())
->get_value_type();
+
+ // Get ORC key and value types with null checks
const orc::Type* orc_key_type = orc_column_type->getSubtype(0);
const orc::Type* orc_value_type = orc_column_type->getSubtype(1);
+
+ VLOG(1) << "[OrcReader] MAP column '" << col_name
+ << "': orc_key_type=" << (orc_key_type != nullptr ? "not null" : "NULL")
+ << ", orc_value_type=" << (orc_value_type != nullptr ? "not null" : "NULL")
+ << ", element_size=" << element_size;
+
+ // Handle incomplete MAP type - if key or value type is nullptr, try to recover from mapping
+ bool key_is_missing = (orc_key_type == nullptr);
+ bool value_is_missing = (orc_value_type == nullptr);
+
+ if (key_is_missing || value_is_missing) {
+ VLOG(1) << "[OrcReader] Detected incomplete MAP subtypes for column '" << col_name
+ << "', attempting to recover from mapping...";
+
+ uint64_t column_id = orc_column_type->getColumnId();
+ auto it = _column_id_to_file_type.find(column_id);
+ if (it != _column_id_to_file_type.end() && it->second != nullptr) {
+ const orc::Type* complete_map_type = it->second;
+ if (complete_map_type->getKind() == orc::TypeKind::MAP &&
+ complete_map_type->getSubtypeCount() == 2) {
+ if (key_is_missing) {
+ orc_key_type = complete_map_type->getSubtype(0);
+ if (orc_key_type != nullptr) {
+ // key_is_missing = false;
+ VLOG(1) << "[OrcReader] Recovered key type from mapping for column '"
+ << col_name << "'";
+ }
+ }
+ if (value_is_missing) {
+ orc_value_type = complete_map_type->getSubtype(1);
+ if (orc_value_type != nullptr) {
+ // value_is_missing = false;
+ VLOG(1) << "[OrcReader] Recovered value type from mapping for column '"
+ << col_name << "'";
+ }
+ }
+ }
+ }
+ }
+
ColumnPtr& doris_key_column = doris_map.get_keys_ptr();
ColumnPtr& doris_value_column = doris_map.get_values_ptr();
std::string key_col_name = col_name + ".key";
std::string value_col_name = col_name + ".value";
- RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
- key_col_name, doris_key_column, doris_key_type, root_node->get_key_node(),
+ // Handle key column: if still missing, fill with default values
+ if (key_is_missing) {
+ LOG(INFO) << "[OrcReader] Key type is missing for MAP column '" << col_name
+ << "', filling with default values (element_size=" << element_size << ")";
+
+ // Fill key column with default values (nulls or empty values)
+ auto mutable_key_column = doris_key_column->assume_mutable();
+ if (mutable_key_column->is_nullable()) {
+ auto* nullable_column = static_cast<ColumnNullable*>(mutable_key_column.get());
+ nullable_column->insert_many_defaults(element_size);
+ } else {
+ mutable_key_column->insert_many_defaults(element_size);
+ }
+ } else {
+ // Normal processing: convert ORC column to Doris column
+ auto status = _orc_column_to_doris_column<false>(
+ key_col_name, doris_key_column, doris_key_type, root_node->get_key_node(),
+ orc_key_type, orc_map->keys.get(), element_size);
+ if (!status.ok()) {
+ return status;
+ }
+ }
+
+ // Handle value column: if still missing, fill with default values
+ if (value_is_missing) {
+ LOG(INFO) << "[OrcReader] Value type is missing for MAP column '" << col_name
+ << "', filling with default values (element_size=" << element_size << ")";
- orc_key_type, orc_map->keys.get(), element_size));
- RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
- value_col_name, doris_value_column, doris_value_type, root_node->get_value_node(),
- orc_value_type, orc_map->elements.get(), element_size));
- return doris_map.deduplicate_keys();
+ // Fill value column with default values (nulls or empty values)
+ auto mutable_value_column = doris_value_column->assume_mutable();
+ if (mutable_value_column->is_nullable()) {
+ auto* nullable_column = static_cast<ColumnNullable*>(mutable_value_column.get());
+ nullable_column->insert_many_defaults(element_size);
+ } else {
+ mutable_value_column->insert_many_defaults(element_size);
+ }
+ return Status::OK();
+ } else {
+ // Normal processing: convert ORC column to Doris column
+ return _orc_column_to_doris_column<false>(value_col_name, doris_value_column,
+ doris_value_type, root_node->get_value_node(),
+ orc_value_type, orc_map->elements.get(),
+ element_size);
+ }
}
case PrimitiveType::TYPE_STRUCT: {
if (orc_column_type->getKind() != orc::TypeKind::STRUCT) {
@@ -1781,6 +2023,14 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const auto* doris_struct_type =
assert_cast<const DataTypeStruct*>(remove_nullable(data_type).get());
+ // Build ORC field name to index map for faster lookup
+ std::unordered_map<std::string, int> orc_field_name_to_idx;
+ for (int j = 0; j < orc_column_type->getSubtypeCount(); ++j) {
+ std::string field_name = orc_column_type->getFieldName(j);
+ std::transform(field_name.begin(), field_name.end(), field_name.begin(), ::tolower);
+ orc_field_name_to_idx[field_name] = j;
+ }
+
for (int i = 0; i < doris_struct.tuple_size(); ++i) {
const auto& table_column_name = doris_struct_type->get_name_by_position(i);
if (!root_node->children_column_exists(table_column_name)) {
@@ -1788,10 +2038,23 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
continue;
}
const auto& file_column_name = root_node->children_file_column_name(table_column_name);
- for (int j = 0; j < orc_column_type->getSubtypeCount(); ++j) {
- if (boost::iequals(orc_column_type->getFieldName(j), file_column_name)) {
- read_fields[i] = j;
- }
+ std::string file_column_name_lower = file_column_name;
+ std::transform(file_column_name_lower.begin(), file_column_name_lower.end(),
+ file_column_name_lower.begin(), ::tolower);
+
+ auto it = orc_field_name_to_idx.find(file_column_name_lower);
+ if (it != orc_field_name_to_idx.end()) {
+ read_fields[i] = it->second;
+ VLOG_DEBUG << "[OrcReader] Found field mapping: doris_field[" << i
+ << "] -> orc_field[" << it->second
+ << "], table_column: " << table_column_name
+ << ", file_column: " << file_column_name_lower;
+ } else {
+ missing_fields.insert(i);
+ VLOG_DEBUG << "[OrcReader] Missing field: doris_field[" << i
+ << "], table_column: " << table_column_name
+ << ", file_column: " << file_column_name_lower
+ << " (not found in ORC file)";
}
}
@@ -1808,6 +2071,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
for (auto read_field : read_fields) {
orc::ColumnVectorBatch* orc_field = orc_struct->fields[read_field.second];
+
const orc::Type* orc_type = orc_column_type->getSubtype(read_field.second);
std::string field_name =
col_name + "." + orc_column_type->getFieldName(read_field.second);
@@ -1868,6 +2132,7 @@ Status OrcReader::_orc_column_to_doris_column(
auto* nullable_column =
reinterpret_cast<ColumnNullable*>(resolved_column->assume_mutable().get());
data_column = nullable_column->get_nested_column_ptr();
+
NullMap& map_data_column = nullable_column->get_null_map_data();
auto origin_size = map_data_column.size();
map_data_column.resize(origin_size + num_values);
@@ -1887,9 +2152,9 @@ Status OrcReader::_orc_column_to_doris_column(
data_column = resolved_column->assume_mutable();
}
- RETURN_IF_ERROR(_fill_doris_data_column<is_filter>(col_name, data_column,
- remove_nullable(resolved_type), root_node,
- orc_column_type, cvb, num_values));
+ RETURN_IF_ERROR((_fill_doris_data_column<is_filter>(col_name, data_column,
+ remove_nullable(resolved_type), root_node,
+ orc_column_type, cvb, num_values)));
// resolve schema change
auto converted_column = doris_column->assume_mutable();
return converter->convert(resolved_column, converted_column);
@@ -1993,10 +2258,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column<true>(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<true>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements));
+ batch_vec[orc_col_idx->second], _batch->numElements)));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2084,10 +2349,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements));
+ batch_vec[orc_col_idx->second], _batch->numElements)));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2250,10 +2515,10 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", table_col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
table_col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(table_col_name),
- _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements));
+ _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements)));
}
RETURN_IF_ERROR(
_fill_partition_columns(block, size, _lazy_read_ctx.predicate_partition_columns));
diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h
index 358ba1d2468..341e40a9ce2 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.h
+++ b/be/src/vec/exec/format/orc/vorc_reader.h
@@ -112,6 +112,12 @@ struct LazyReadContext {
std::unordered_map<std::string, VExprContextSPtr> predicate_missing_columns;
// lazy read missing columns or all missing columns
std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+
+ std::vector<std::string> partial_predicate_columns;
+
+ // Record the number of rows filled in filter phase for lazy materialization
+ // This is used to check if a column was already processed in filter phase
+ size_t filter_phase_rows = 0;
};
class OrcReader : public GenericReader {
@@ -158,7 +164,9 @@ public:
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr =
- TableSchemaChangeHelper::ConstNode::get_instance());
+ TableSchemaChangeHelper::ConstNode::get_instance(),
+ const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
Status set_fill_columns(
const std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>&
@@ -195,7 +203,8 @@ public:
std::unordered_map<std::string, orc::StringDictionary*>& column_name_to_dict_map,
bool* is_stripe_filtered);
- static DataTypePtr convert_to_doris_type(const orc::Type* orc_type);
+ DataTypePtr convert_to_doris_type(const orc::Type* orc_type);
+
static std::string get_field_name_lower_case(const orc::Type* orc_type, int pos);
void set_row_id_column_iterator(
@@ -630,7 +639,8 @@ private:
size_t _batch_size;
int64_t _range_start_offset;
int64_t _range_size;
- const std::string& _ctz;
+ // const std::string& _ctz;
+ std::string _ctz;
int32_t _offset_days = 0;
cctz::time_zone _time_zone;
@@ -653,6 +663,12 @@ private:
// file column name to orc type
std::unordered_map<std::string, const orc::Type*> _type_map;
+ // Optimized type conversion support
+ // column ID to file original type mapping for handling partial column selection
+ std::unordered_map<uint64_t, const orc::Type*> _column_id_to_file_type;
+ // Keep reference to file root type for complete type information
+ const orc::Type* _file_root_type = nullptr;
+
std::unique_ptr<ORCFileInputStream> _file_input_stream;
Statistics _statistics;
OrcProfile _orc_profile;
@@ -673,6 +689,8 @@ private:
std::vector<DecimalScaleParams> _decimal_scale_params;
size_t _decimal_scale_params_index;
+ std::set<uint64_t> _column_ids;
+ std::set<uint64_t> _filter_column_ids;
const std::unordered_map<std::string, ColumnValueRangeType>* _colname_to_value_range = nullptr;
bool _is_acid = false;
std::unique_ptr<IColumn::Filter> _filter;
diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp
index 82e6e0c9d61..c022f1e3cfe 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.cpp
+++ b/be/src/vec/exec/format/parquet/schema_desc.cpp
@@ -176,6 +176,7 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
parse_physical_field(t_schema, false, child);
node_field->name = t_schema.name;
+ node_field->lower_case_name = to_lower(t_schema.name);
node_field->data_type = std::make_shared<DataTypeArray>(make_nullable(child->data_type));
_next_schema_pos = curr_pos + 1;
node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1;
@@ -193,8 +194,10 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
void FieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema,
bool is_nullable, FieldSchema* physical_field) {
physical_field->name = physical_schema.name;
+ physical_field->lower_case_name = to_lower(physical_field->name);
physical_field->parquet_schema = physical_schema;
physical_field->physical_type = physical_schema.type;
+ physical_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
_physical_fields.push_back(physical_field);
physical_field->physical_column_index = cast_set<int>(_physical_fields.size() - 1);
auto type = get_doris_type(physical_schema, is_nullable);
@@ -393,6 +396,8 @@ Status FieldDescriptor::parse_group_field(const std::vector<tparquet::SchemaElem
RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field));
group_field->name = group_schema.name;
+ group_field->lower_case_name = to_lower(group_field->name);
+ group_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
group_field->data_type =
std::make_shared<DataTypeArray>(make_nullable(struct_field->data_type));
group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1;
@@ -461,6 +466,8 @@ Status FieldDescriptor::parse_list_field(const std::vector<tparquet::SchemaEleme
}
list_field->name = first_level.name;
+ list_field->lower_case_name = to_lower(first_level.name);
+ list_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
list_field->data_type =
std::make_shared<DataTypeArray>(make_nullable(list_field->children[0].data_type));
if (is_optional) {
@@ -519,21 +526,24 @@ Status FieldDescriptor::parse_map_field(const std::vector<tparquet::SchemaElemen
map_field->repetition_level++;
map_field->definition_level++;
- map_field->children.resize(1);
+ // Directly create key and value children instead of intermediate key_value node
+ map_field->children.resize(2);
// map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level`
set_child_node_level(map_field, map_field->definition_level);
- auto map_kv_field = &map_field->children[0];
- // produce MAP<STRUCT<KEY, VALUE>>
- RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, map_kv_field));
+
+ auto key_field = &map_field->children[0];
+ auto value_field = &map_field->children[1];
+
+ // Parse key and value fields directly from the key_value group's children
+ _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key
+ RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field));
+ RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field));
map_field->name = map_schema.name;
- map_field->data_type = std::make_shared<DataTypeMap>(
- make_nullable(assert_cast<const DataTypeStruct*>(
- remove_nullable(map_kv_field->data_type).get())
- ->get_element(0)),
- make_nullable(assert_cast<const DataTypeStruct*>(
- remove_nullable(map_kv_field->data_type).get())
- ->get_element(1)));
+ map_field->lower_case_name = to_lower(map_field->name);
+ map_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
+ map_field->data_type = std::make_shared<DataTypeMap>(make_nullable(key_field->data_type),
+ make_nullable(value_field->data_type));
if (is_optional) {
map_field->data_type = make_nullable(map_field->data_type);
}
@@ -558,6 +568,8 @@ Status FieldDescriptor::parse_struct_field(const std::vector<tparquet::SchemaEle
RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i]));
}
struct_field->name = struct_schema.name;
+ struct_field->lower_case_name = to_lower(struct_field->name);
+ struct_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1;
DataTypes res_data_types;
@@ -610,6 +622,59 @@ std::string FieldDescriptor::debug_string() const {
ss << "]";
return ss.str();
}
+
+void FieldDescriptor::assign_ids() {
+ uint64_t next_id = 1;
+ for (auto& field : _fields) {
+ field.assign_ids(next_id);
+ }
+}
+
+const FieldSchema* FieldDescriptor::find_column_by_id(uint64_t column_id) const {
+ for (const auto& field : _fields) {
+ if (auto result = field.find_column_by_id(column_id)) {
+ return result;
+ }
+ }
+ return nullptr;
+}
+
+void FieldSchema::assign_ids(uint64_t& next_id) {
+ column_id = next_id++;
+
+ for (auto& child : children) {
+ child.assign_ids(next_id);
+ }
+
+ max_column_id = next_id - 1;
+}
+
+const FieldSchema* FieldSchema::find_column_by_id(uint64_t target_id) const {
+ if (column_id == target_id) {
+ return this;
+ }
+
+ for (const auto& child : children) {
+ if (auto result = child.find_column_by_id(target_id)) {
+ return result;
+ }
+ }
+
+ return nullptr;
+}
+
+uint64_t FieldSchema::get_column_id() const {
+ return column_id;
+}
+
+void FieldSchema::set_column_id(uint64_t id) {
+ column_id = id;
+}
+
+uint64_t FieldSchema::get_max_column_id() const {
+ return max_column_id;
+}
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/schema_desc.h b/be/src/vec/exec/format/parquet/schema_desc.h
index 09a52a4251a..3ba38071406 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.h
+++ b/be/src/vec/exec/format/parquet/schema_desc.h
@@ -36,8 +36,13 @@
namespace doris::vectorized {
#include "common/compile_check_begin.h"
+
+// Constant for unassigned column IDs
+constexpr uint64_t UNASSIGNED_COLUMN_ID = UINT64_MAX;
+
struct FieldSchema {
std::string name;
+ std::string lower_case_name; // for hms column name case insensitive match
// the referenced parquet schema element
tparquet::SchemaElement parquet_schema;
@@ -56,12 +61,22 @@ struct FieldSchema {
//For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128.
bool is_type_compatibility = false;
- FieldSchema() : data_type(std::make_shared<DataTypeNothing>()) {}
+ FieldSchema()
+ : data_type(std::make_shared<DataTypeNothing>()), column_id(UNASSIGNED_COLUMN_ID) {}
~FieldSchema() = default;
FieldSchema(const FieldSchema& fieldSchema) = default;
std::string debug_string() const;
int32_t field_id = -1;
+ uint64_t column_id = UNASSIGNED_COLUMN_ID;
+ uint64_t max_column_id = 0; // Maximum column ID for this field and its children
+
+ // Column ID assignment and lookup methods
+ void assign_ids(uint64_t& next_id);
+ const FieldSchema* find_column_by_id(uint64_t target_id) const;
+ uint64_t get_column_id() const;
+ void set_column_id(uint64_t id);
+ uint64_t get_max_column_id() const;
};
class FieldDescriptor {
@@ -133,6 +148,19 @@ public:
int32_t size() const { return cast_set<int32_t>(_fields.size()); }
const std::vector<FieldSchema>& get_fields_schema() const { return _fields; }
+
+ /**
+ * Assign stable column IDs to schema fields.
+ *
+ * This uses an ORC-compatible encoding so that the results of
+ * create_column_ids() are consistent across formats. IDs start from 1
+ * and are assigned in a pre-order traversal (parent before children).
+ * After calling this, each FieldSchema will have column_id and
+ * max_column_id populated.
+ */
+ void assign_ids();
+
+ const FieldSchema* find_column_by_id(uint64_t column_id) const;
};
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index 74fe4c3601b..0ceecf1f610 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -108,26 +108,52 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
const std::vector<RowRange>& row_ranges,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
std::unique_ptr<ParquetColumnReader>& reader,
- size_t max_buf_size, const tparquet::OffsetIndex* offset_index) {
+ size_t max_buf_size, const tparquet::OffsetIndex* offset_index,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ if (!column_ids.empty()) {
+ uint64_t field_column_id = field->get_column_id();
+ VLOG_DEBUG << "[ParquetReader] Checking column_id: " << field_column_id
+ << ", column_ids.size(): " << column_ids.size();
+ if (column_ids.find(field_column_id) == column_ids.end()) {
+ VLOG_DEBUG << "[ParquetReader] Setting skip_reading=true for column_id: "
+ << field_column_id;
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field);
+ skip_reader->_filter_column_ids = filter_column_ids;
+ reader = std::move(skip_reader);
+ return Status::OK();
+ } else {
+ VLOG_DEBUG << "[ParquetReader] Column_id " << field_column_id
+ << " found in column_ids, skip_reading=false";
+ }
+ }
+
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
std::unique_ptr<ParquetColumnReader> element_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
- element_reader, max_buf_size));
+ element_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
element_reader->set_nested_column();
auto array_reader = ArrayColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field));
+ array_reader->_filter_column_ids = filter_column_ids;
reader.reset(array_reader.release());
} else if (field->data_type->get_primitive_type() == TYPE_MAP) {
std::unique_ptr<ParquetColumnReader> key_reader;
std::unique_ptr<ParquetColumnReader> value_reader;
- RETURN_IF_ERROR(create(file, &field->children[0].children[0], row_group, row_ranges, ctz,
- io_ctx, key_reader, max_buf_size));
- RETURN_IF_ERROR(create(file, &field->children[0].children[1], row_group, row_ranges, ctz,
- io_ctx, value_reader, max_buf_size));
+
+ // Create key reader
+ RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
+ key_reader, max_buf_size, nullptr, column_ids, filter_column_ids));
+
+ // Create value reader
+ RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx,
+ value_reader, max_buf_size, nullptr, column_ids, filter_column_ids));
key_reader->set_nested_column();
value_reader->set_nested_column();
auto map_reader = MapColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field));
+ map_reader->_filter_column_ids = filter_column_ids;
reader.reset(map_reader.release());
} else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
std::unordered_map<std::string, std::unique_ptr<ParquetColumnReader>> child_readers;
@@ -135,18 +161,21 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
for (int i = 0; i < field->children.size(); ++i) {
std::unique_ptr<ParquetColumnReader> child_reader;
RETURN_IF_ERROR(create(file, &field->children[i], row_group, row_ranges, ctz, io_ctx,
- child_reader, max_buf_size));
+ child_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
child_reader->set_nested_column();
child_readers[field->children[i].name] = std::move(child_reader);
}
auto struct_reader = StructColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field));
+ struct_reader->_filter_column_ids = filter_column_ids;
reader.reset(struct_reader.release());
} else {
const tparquet::ColumnChunk& chunk = row_group.columns[field->physical_column_index];
auto scalar_reader =
ScalarColumnReader::create_unique(row_ranges, chunk, offset_index, ctz, io_ctx);
RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ scalar_reader->_filter_column_ids = filter_column_ids;
reader.reset(scalar_reader.release());
}
return Status::OK();
@@ -851,6 +880,7 @@ Status StructColumnReader::read_column_data(
int64_t not_missing_column_id = -1;
std::vector<size_t> missing_column_idxs {};
+ std::vector<size_t> skip_reading_column_idxs {};
_read_column_names.clear();
@@ -860,10 +890,29 @@ Status StructColumnReader::read_column_data(
auto& doris_name = doris_struct_type->get_element_name(i);
if (!root_node->children_column_exists(doris_name)) {
missing_column_idxs.push_back(i);
+ VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i
+ << "], doris_name: " << doris_name << " (column not exists in root node)";
continue;
}
auto file_name = root_node->children_file_column_name(doris_name);
+ // if (_child_readers.find(file_name) == _child_readers.end()) {
+ // missing_column_idxs.push_back(i);
+ // continue;
+ // }
+
+ // Check if this is a SkipReadingReader - we should skip it when choosing reference column
+ // because SkipReadingReader doesn't know the actual data size in nested context
+ bool is_skip_reader =
+ dynamic_cast<SkipReadingReader*>(_child_readers[file_name].get()) != nullptr;
+
+ if (is_skip_reader) {
+ // Store SkipReadingReader columns to fill them later based on reference column size
+ skip_reading_column_idxs.push_back(i);
+ continue;
+ }
+ // Only add non-SkipReadingReader columns to _read_column_names
+ // This ensures get_rep_level() and get_def_level() return valid levels
_read_column_names.emplace_back(file_name);
size_t field_rows = 0;
@@ -913,7 +962,23 @@ Status StructColumnReader::read_column_data(
// When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2.
// You should insert 5 records into subcolumn b instead of 2.
auto missing_column_sz = doris_struct.get_column(not_missing_column_id).size();
- // fill missing column with null or default value
+
+ // Fill SkipReadingReader columns with the correct amount of data based on reference column
+ // Let SkipReadingReader handle the data filling through its read_column_data method
+ for (auto idx : skip_reading_column_idxs) {
+ auto& doris_field = doris_struct.get_column_ptr(idx);
+ auto& doris_type = const_cast<DataTypePtr&>(doris_struct_type->get_element(idx));
+ auto& doris_name = const_cast<String&>(doris_struct_type->get_element_name(idx));
+ auto file_name = root_node->children_file_column_name(doris_name);
+
+ size_t field_rows = 0;
+ bool field_eof = false;
+ RETURN_IF_ERROR(_child_readers[file_name]->read_column_data(
+ doris_field, doris_type, root_node->get_children_node(doris_name), filter_map,
+ missing_column_sz, &field_rows, &field_eof, is_dict_filter));
+ }
+
+ // Fill truly missing columns (not in root_node) with null or default value
for (auto idx : missing_column_idxs) {
auto& doris_field = doris_struct.get_column_ptr(idx);
auto& doris_type = doris_struct_type->get_element(idx);
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index f0d02b0d3e4..05b47403291 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -138,7 +138,9 @@ public:
const tparquet::RowGroup& row_group,
const std::vector<RowRange>& row_ranges, const cctz::time_zone* ctz,
io::IOContext* io_ctx, std::unique_ptr<ParquetColumnReader>& reader,
- size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr);
+ size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr,
+ const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
void set_nested_column() { _nested_column = true; }
virtual const std::vector<level_t>& get_rep_level() const = 0;
virtual const std::vector<level_t>& get_def_level() const = 0;
@@ -147,6 +149,8 @@ public:
virtual void reset_filter_map_index() = 0;
+ FieldSchema* get_field_schema() const { return _field_schema; }
+
protected:
void _generate_read_ranges(int64_t start_index, int64_t end_index,
std::list<RowRange>& read_ranges);
@@ -162,6 +166,7 @@ protected:
int64_t _decode_null_map_time = 0;
size_t _filter_map_index = 0;
+ std::set<uint64_t> _filter_column_ids;
};
class ScalarColumnReader : public ParquetColumnReader {
@@ -346,6 +351,90 @@ private:
std::vector<std::string> _read_column_names;
//Need to use vector instead of set,see `get_rep_level()` for the reason.
};
+
+// A special reader that skips actual reading but provides empty data with correct structure
+// This is used when a column is not needed but its structure is required (e.g., for map keys)
+class SkipReadingReader : public ParquetColumnReader {
+public:
+ SkipReadingReader(const std::vector<RowRange>& row_ranges, const cctz::time_zone* ctz,
+ io::IOContext* io_ctx, FieldSchema* field_schema)
+ : ParquetColumnReader(row_ranges, ctz, io_ctx) {
+ _field_schema = field_schema; // Use inherited member from base class
+ VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: "
+ << _field_schema->name;
+ }
+
+ Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
+ FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
+ bool is_dict_filter) override {
+ VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: "
+ << _field_schema->name << ", batch_size: " << batch_size;
+
+ // Simulate reading without actually reading data
+ // Fill with default/null values based on column type
+ MutableColumnPtr data_column = doris_column->assume_mutable();
+
+ if (doris_column->is_nullable()) {
+ auto* nullable_column = static_cast<vectorized::ColumnNullable*>(data_column.get());
+ nullable_column->insert_many_defaults(batch_size);
+ } else {
+ // For non-nullable columns, insert appropriate default values
+ for (size_t i = 0; i < batch_size; ++i) {
+ data_column->insert_default();
+ }
+ }
+
+ *read_rows = batch_size;
+ *eof = false; // We can always provide more empty data
+
+ VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size
+ << " default values for field: " << _field_schema->name;
+
+ return Status::OK();
+ }
+
+ static std::unique_ptr<SkipReadingReader> create_unique(const std::vector<RowRange>& row_ranges,
+ cctz::time_zone* ctz,
+ io::IOContext* io_ctx,
+ FieldSchema* field_schema) {
+ return std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field_schema);
+ }
+
+ // These methods should not be called for SkipReadingReader
+ // If they are called, it indicates a logic error in the code
+ const std::vector<level_t>& get_rep_level() const override {
+ LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: "
+ << _field_schema->name
+ << ". This indicates the SkipReadingReader was incorrectly used as a reference "
+ "column.";
+ // static std::vector<level_t> empty_levels;
+ // return empty_levels;
+ __builtin_unreachable();
+ }
+
+ const std::vector<level_t>& get_def_level() const override {
+ LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: "
+ << _field_schema->name
+ << ". This indicates the SkipReadingReader was incorrectly used as a reference "
+ "column.";
+ // static std::vector<level_t> empty_levels;
+ // return empty_levels;
+ __builtin_unreachable();
+ }
+
+ // Implement required pure virtual methods from base class
+ Statistics statistics() override {
+ return Statistics(); // Return empty statistics
+ }
+
+ void close() override {
+ // Nothing to close for skip reading
+ }
+
+ void reset_filter_map_index() override { _filter_map_index = 0; }
+};
+
#include "common/compile_check_end.h"
}; // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index 7d1f9d27430..4094448aa24 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -82,7 +82,9 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
const int32_t row_group_id, const tparquet::RowGroup& row_group,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
const PositionDeleteContext& position_delete_ctx,
- const LazyReadContext& lazy_read_ctx, RuntimeState* state)
+ const LazyReadContext& lazy_read_ctx, RuntimeState* state,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids)
: _file_reader(file_reader),
_read_table_columns(read_columns),
_row_group_id(row_group_id),
@@ -93,7 +95,9 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
_position_delete_ctx(position_delete_ctx),
_lazy_read_ctx(lazy_read_ctx),
_state(state),
- _obj_pool(new ObjectPool()) {}
+ _obj_pool(new ObjectPool()),
+ _column_ids(column_ids),
+ _filter_column_ids(filter_column_ids) {}
RowGroupReader::~RowGroupReader() {
_column_readers.clear();
@@ -130,9 +134,9 @@ Status RowGroupReader::init(
const tparquet::OffsetIndex* offset_index =
col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index]
: nullptr;
- RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field, _row_group_meta,
- _read_ranges, _ctz, _io_ctx, reader,
- max_buf_size, offset_index));
+ RETURN_IF_ERROR(ParquetColumnReader::create(
+ _file_reader, field, _row_group_meta, _read_ranges, _ctz, _io_ctx, reader,
+ max_buf_size, offset_index, _column_ids, _filter_column_ids));
if (reader == nullptr) {
VLOG_DEBUG << "Init row group(" << _row_group_id << ") reader failed";
return Status::Corruption("Init row group reader failed");
@@ -320,8 +324,8 @@ Status RowGroupReader::next_batch(Block* block, size_t batch_size, size_t* read_
return _do_lazy_read(block, batch_size, read_rows, batch_eof);
} else {
FilterMap filter_map;
- RETURN_IF_ERROR(_read_column_data(block, _lazy_read_ctx.all_read_columns, batch_size,
- read_rows, batch_eof, filter_map));
+ RETURN_IF_ERROR((_read_column_data(block, _lazy_read_ctx.all_read_columns, batch_size,
+ read_rows, batch_eof, filter_map)));
RETURN_IF_ERROR(
_fill_partition_columns(block, *read_rows, _lazy_read_ctx.partition_columns));
RETURN_IF_ERROR(_fill_missing_columns(block, *read_rows, _lazy_read_ctx.missing_columns));
@@ -1110,6 +1114,7 @@ ParquetColumnReader::Statistics RowGroupReader::statistics() {
}
return st;
}
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
index 86b7cdf3f0e..5853cd6827d 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
@@ -150,7 +150,9 @@ public:
const int32_t row_group_id, const tparquet::RowGroup& row_group,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
const PositionDeleteContext& position_delete_ctx,
- const LazyReadContext& lazy_read_ctx, RuntimeState* state);
+ const LazyReadContext& lazy_read_ctx, RuntimeState* state,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids);
~RowGroupReader();
Status init(const FieldDescriptor& schema, std::vector<RowRange>& row_ranges,
@@ -188,9 +190,11 @@ private:
void _merge_read_ranges(std::vector<RowRange>& row_ranges);
Status _read_empty_batch(size_t batch_size, size_t* read_rows, bool* batch_eof,
bool* modify_row_ids);
+
Status _read_column_data(Block* block, const std::vector<std::string>& columns,
size_t batch_size, size_t* read_rows, bool* batch_eof,
FilterMap& filter_map);
+
Status _do_lazy_read(Block* block, size_t batch_size, size_t* read_rows, bool* batch_eof);
Status _rebuild_filter_map(FilterMap& filter_map,
DorisUniqueBufferPtr<uint8_t>& filter_map_data,
@@ -249,6 +253,8 @@ private:
std::vector<std::pair<std::string, int>> _dict_filter_cols;
RuntimeState* _state = nullptr;
std::shared_ptr<ObjectPool> _obj_pool;
+ const std::set<uint64_t>& _column_ids;
+ const std::set<uint64_t>& _filter_column_ids;
bool _is_row_group_filtered = false;
RowGroupIndex _current_row_group_idx {0, 0, 0};
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 9a4ac82c5d2..27c2bdded28 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -319,7 +319,8 @@ Status ParquetReader::init_reader(
const std::unordered_map<std::string, int>* colname_to_slot_id,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
- std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr, bool filter_groups) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr, bool filter_groups,
+ const std::set<uint64_t>& column_ids, const std::set<uint64_t>& filter_column_ids) {
_tuple_descriptor = tuple_descriptor;
_row_descriptor = row_descriptor;
_colname_to_slot_id = colname_to_slot_id;
@@ -328,6 +329,8 @@ Status ParquetReader::init_reader(
_colname_to_value_range = colname_to_value_range;
_table_info_node_ptr = table_info_node_ptr;
_filter_groups = filter_groups;
+ _column_ids = column_ids;
+ _filter_column_ids = filter_column_ids;
RETURN_IF_ERROR(_open_file());
_t_metadata = &(_file_metadata->to_thrift());
@@ -878,12 +881,12 @@ Status ParquetReader::_next_row_group_reader() {
_profile, _file_reader, io_ranges, merged_read_slice_size)
: _file_reader;
}
- _current_group_reader.reset(
- new RowGroupReader(_io_ctx ? std::make_shared<io::TracingFileReader>(
- group_file_reader, _io_ctx->file_reader_stats)
- : group_file_reader,
- _read_table_columns, row_group_index.row_group_id, row_group, _ctz,
- _io_ctx, position_delete_ctx, _lazy_read_ctx, _state));
+ _current_group_reader.reset(new RowGroupReader(
+ _io_ctx ? std::make_shared<io::TracingFileReader>(group_file_reader,
+ _io_ctx->file_reader_stats)
+ : group_file_reader,
+ _read_table_columns, row_group_index.row_group_id, row_group, _ctz, _io_ctx,
+ position_delete_ctx, _lazy_read_ctx, _state, _column_ids, _filter_column_ids));
_row_group_eof = false;
_current_group_reader->set_current_row_group_idx(row_group_index);
@@ -968,8 +971,8 @@ std::vector<io::PrefetchRange> ParquetReader::_generate_random_access_ranges(
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
scalar_range(&field->children[0], row_group);
} else if (field->data_type->get_primitive_type() == TYPE_MAP) {
- scalar_range(&field->children[0].children[0], row_group);
- scalar_range(&field->children[0].children[1], row_group);
+ scalar_range(&field->children[0], row_group);
+ scalar_range(&field->children[1], row_group);
} else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
for (int i = 0; i < field->children.size(); ++i) {
scalar_range(&field->children[i], row_group);
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h
index 3b9a6b04db9..5beef4a9358 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.h
@@ -123,7 +123,8 @@ public:
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr =
TableSchemaChangeHelper::ConstNode::get_instance(),
- bool filter_groups = true);
+ bool filter_groups = true, const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
Status get_next_block(Block* block, size_t* read_rows, bool* eof) override;
@@ -324,6 +325,8 @@ private:
-1};
bool _filter_groups;
+ std::set<uint64_t> _column_ids;
+ std::set<uint64_t> _filter_column_ids;
// push down =, >, <, >=, <=, in
VExprSPtrs _push_down_exprs;
diff --git a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
new file mode 100644
index 00000000000..086a091400a
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive/hive_orc_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/logging.h"
+#include "orc/Type.hh"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void HiveOrcNestedColumnUtils::extract_nested_column_ids(
+ const orc::Type& type, const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<TColumnNameAccessPath>>
+ child_paths_by_table_col_name;
+
+ for (const auto& access_path : paths) {
+ if (!access_path.path.empty()) {
+ std::string first_table_name = access_path.path[0];
+ TColumnNameAccessPath remaining;
+ if (access_path.path.size() > 1) {
+ remaining.path.assign(access_path.path.begin() + 1, access_path.path.end());
+ }
+ child_paths_by_table_col_name[first_table_name].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (type.getKind() == orc::TypeKind::MAP) {
+ auto wildcard_it = child_paths_by_table_col_name.find("*");
+ if (wildcard_it != child_paths_by_table_col_name.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_table_col_name["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_table_col_name["KEYS"];
+ // Add an empty path to request full KEYS
+ TColumnNameAccessPath empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_table_col_name.erase(wildcard_it);
+ }
+ }
+
+ for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) {
+ const orc::Type* child = type.getSubtype(i);
+
+ std::string child_field_name;
+ switch (type.getKind()) {
+ case orc::TypeKind::STRUCT:
+ child_field_name = to_lower(type.getFieldName(i));
+ break;
+ case orc::TypeKind::LIST:
+ child_field_name = "*";
+ break;
+ case orc::TypeKind::MAP:
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_name = "KEYS";
+ } else if (i == 1) {
+ child_field_name = "VALUES";
+ }
+ break;
+ default:
+ child_field_name = "";
+ break;
+ }
+
+ if (child_field_name.empty()) {
+ continue;
+ }
+
+ const auto child_paths_it = child_paths_by_table_col_name.find(child_field_name);
+
+ if (child_paths_it != child_paths_by_table_col_name.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child = std::any_of(child_paths.begin(), child_paths.end(),
+ [](const TColumnNameAccessPath& access_path) {
+ return access_path.path.empty();
+ });
+
+ if (needs_full_child) {
+ uint64_t start_id = child->getColumnId();
+ uint64_t max_column_id = child->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(*child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ };
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(type.getColumnId());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h
new file mode 100644
index 00000000000..d7592cab003
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace orc {
+class Type;
+} // namespace orc
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class HiveOrcNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const orc::Type& type,
+ const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp
new file mode 100644
index 00000000000..a4d182b3311
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive/hive_parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/parquet/schema_desc.h"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void HiveParquetNestedColumnUtils::extract_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<TColumnNameAccessPath>>
+ child_paths_by_table_col_name;
+
+ for (const auto& access_path : paths) {
+ if (!access_path.path.empty()) {
+ std::string first_table_col_name = access_path.path[0];
+ TColumnNameAccessPath remaining;
+ if (access_path.path.size() > 1) {
+ remaining.path.assign(access_path.path.begin() + 1, access_path.path.end());
+ }
+ child_paths_by_table_col_name[first_table_col_name].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP) {
+ auto wildcard_it = child_paths_by_table_col_name.find("*");
+ if (wildcard_it != child_paths_by_table_col_name.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_table_col_name["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_table_col_name["KEYS"];
+ // Add an empty path to request full KEYS
+ TColumnNameAccessPath empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_table_col_name.erase(wildcard_it);
+ }
+ }
+
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < field_schema.children.size(); ++i) {
+ const auto& child = field_schema.children[i];
+ std::string child_field_name;
+
+ bool is_list = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY;
+ bool is_map = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP;
+
+ if (is_list) {
+ child_field_name = "*";
+ } else if (is_map) {
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_name = "KEYS";
+ } else if (i == 1) {
+ child_field_name = "VALUES";
+ }
+
+ // Special handling for Parquet MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access = child_paths_by_table_col_name.find("KEYS") !=
+ child_paths_by_table_col_name.end();
+ bool has_values_access = child_paths_by_table_col_name.find("VALUES") !=
+ child_paths_by_table_col_name.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for RL/DL
+ if (!has_keys_access && has_values_access) {
+ // For map_values() queries, we need key's structure for correct RL/DL parsing.
+ // If key is a nested type (e.g., STRUCT), RL/DL info is stored at leaf columns.
+ // Add all column IDs from key's start to max (all leaves + intermediate nodes).
+ uint64_t key_start_id = child.get_column_id();
+ uint64_t key_max_id = child.get_max_column_id();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
+
+ } else {
+ child_field_name = child.lower_case_name;
+ }
+
+ if (child_field_name.empty()) {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_table_col_name.find(child_field_name);
+ if (child_paths_it != child_paths_by_table_col_name.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child = std::any_of(child_paths.begin(), child_paths.end(),
+ [](const TColumnNameAccessPath& access_path) {
+ return access_path.path.empty();
+ });
+
+ if (needs_full_child) {
+ // Add all column IDs from current child node to max_column_id
+ // This efficiently handles all nested/complex cases in one loop
+ uint64_t start_id = child.get_column_id();
+ uint64_t max_column_id = child.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(field_schema.get_column_id());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h
new file mode 100644
index 00000000000..c7141fa2efd
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+struct FieldSchema;
+
+class HiveParquetNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const FieldSchema& field_schema,
+ const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive_reader.cpp b/be/src/vec/exec/format/table/hive_reader.cpp
index a454118b244..527dacbc3ac 100644
--- a/be/src/vec/exec/format/table/hive_reader.cpp
+++ b/be/src/vec/exec/format/table/hive_reader.cpp
@@ -21,6 +21,8 @@
#include "common/status.h"
#include "runtime/runtime_state.h"
+#include "vec/exec/format/table/hive/hive_orc_nested_column_utils.h"
+#include "vec/exec/format/table/hive/hive_parquet_nested_column_utils.h"
namespace doris::vectorized {
#include "common/compile_check_begin.h"
@@ -77,10 +79,213 @@ Status HiveOrcReader::init_reader(
}
}
+ auto column_id_result = ColumnIdResult();
+ if (_state->query_options().hive_orc_use_column_names && !is_hive_col_name) {
+ column_id_result = _create_column_ids(orc_type_ptr, tuple_descriptor);
+ } else {
+ column_id_result =
+ _create_column_ids_by_top_level_col_index(orc_type_ptr, tuple_descriptor);
+ }
+
+ // const auto& file_col_names = column_id_result.column_names;
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
return orc_reader->init_reader(&read_table_col_names, table_col_name_to_value_range, conjuncts,
false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts,
- table_info_node_ptr);
+ table_info_node_ptr, column_ids, filter_column_ids);
+}
+
+ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column name (lower-cased) -> orc::Type*
+ std::unordered_map<std::string, const orc::Type*> table_col_name_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ std::string table_col_name = to_lower(orc_type->getFieldName(i));
+ table_col_name_to_orc_type_map[table_col_name] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnNameAccessPath>& name_access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (name_access_paths.empty()) return;
+
+ std::vector<TColumnNameAccessPath> paths;
+ bool has_top_level_only = false;
+ for (const auto& name_access_path : name_access_paths) {
+ DCHECK(name_access_path.path.size() >= 1);
+ TColumnNameAccessPath remaining_path;
+ if (name_access_path.path.size() > 1) {
+ remaining_path.path.assign(name_access_path.path.begin() + 1,
+ name_access_path.path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path.path = std::vector<std::string>();
+ }
+ if (remaining_path.path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ HiveOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_name_to_orc_type_map.find(slot->col_name());
+ if (it == table_col_name_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_column_access_paths = slot->all_column_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_column_access_paths -> column_ids
+ if (all_column_access_paths.__isset.name_access_paths &&
+ !all_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(orc_field, all_column_access_paths.name_access_paths, column_ids);
+ }
+
+ // collect and process predicate_column_access_paths -> filter_column_ids
+ const auto& predicate_column_access_paths = slot->predicate_column_access_paths();
+ if (predicate_column_access_paths.__isset.name_access_paths &&
+ !predicate_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_column_access_paths.name_access_paths,
+ filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
+}
+
+ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
+ const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column index -> orc::Type*
+ std::unordered_map<uint64_t, const orc::Type*> table_col_pos_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ table_col_pos_to_orc_type_map[i] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnNameAccessPath>& name_access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (name_access_paths.empty()) return;
+
+ std::vector<TColumnNameAccessPath> paths;
+ bool has_top_level_only = false;
+ for (const auto& name_access_path : name_access_paths) {
+ DCHECK(name_access_path.path.size() >= 1);
+ TColumnNameAccessPath remaining_path;
+ if (name_access_path.path.size() > 1) {
+ remaining_path.path.assign(name_access_path.path.begin() + 1,
+ name_access_path.path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path.path = std::vector<std::string>();
+ }
+ if (remaining_path.path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ HiveOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_pos_to_orc_type_map.find(slot->col_pos());
+ if (it == table_col_pos_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_column_access_paths = slot->all_column_access_paths();
+
+ // primitive (non-nested) types: direct mapping by pos
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // complex types
+
+ // collect and process all_column_access_paths -> column_ids
+ if (all_column_access_paths.__isset.name_access_paths &&
+ !all_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(orc_field, all_column_access_paths.name_access_paths, column_ids);
+ }
+
+ // collect and process predicate_column_access_paths -> filter_column_ids
+ const auto& predicate_column_access_paths = slot->predicate_column_access_paths();
+ if (predicate_column_access_paths.__isset.name_access_paths &&
+ !predicate_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_column_access_paths.name_access_paths,
+ filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status HiveParquetReader::init_reader(
@@ -145,10 +350,230 @@ Status HiveParquetReader::init_reader(
}
}
- return parquet_reader->init_reader(read_table_col_names, table_col_name_to_value_range,
- conjuncts, tuple_descriptor, row_descriptor,
- colname_to_slot_id, not_single_slot_filter_conjuncts,
- slot_id_to_filter_conjuncts, table_info_node_ptr);
+ auto column_id_result = ColumnIdResult();
+ if (_state->query_options().hive_parquet_use_column_names) {
+ column_id_result = _create_column_ids(field_desc, tuple_descriptor);
+ } else {
+ column_id_result = _create_column_ids_by_top_level_col_index(field_desc, tuple_descriptor);
+ }
+
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
+ RETURN_IF_ERROR(init_row_filters());
+
+ return parquet_reader->init_reader(
+ read_table_col_names, table_col_name_to_value_range, conjuncts, tuple_descriptor,
+ row_descriptor, colname_to_slot_id, not_single_slot_filter_conjuncts,
+ slot_id_to_filter_conjuncts, table_info_node_ptr, true, column_ids, filter_column_ids);
+}
+
+ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<std::string, const FieldSchema*> table_col_name_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ table_col_name_to_field_schema_map[field_schema->lower_case_name] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnNameAccessPath>& name_access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (name_access_paths.empty()) return;
+
+ std::vector<TColumnNameAccessPath> paths;
+ bool has_top_level_only = false;
+ for (const auto& name_access_path : name_access_paths) {
+ DCHECK(name_access_path.path.size() >= 1);
+ TColumnNameAccessPath remaining_path;
+ if (name_access_path.path.size() > 1) {
+ remaining_path.path.assign(name_access_path.path.begin() + 1,
+ name_access_path.path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path.path = std::vector<std::string>();
+ }
+ if (remaining_path.path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ HiveParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+
+ // Find the field schema for this slot (may not exist for partition columns, etc.)
+ auto it = table_col_name_to_field_schema_map.find(slot->col_name());
+ if (it == table_col_name_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_column_access_paths = slot->all_column_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_column_access_paths -> column_ids
+ if (all_column_access_paths.__isset.name_access_paths &&
+ !all_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(field_schema, all_column_access_paths.name_access_paths,
+ column_ids);
+ }
+
+ // collect and process predicate_column_access_paths -> filter_column_ids
+ const auto& predicate_column_access_paths = slot->predicate_column_access_paths();
+ if (predicate_column_access_paths.__isset.name_access_paths &&
+ !predicate_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_column_access_paths.name_access_paths,
+ filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
+}
+
+ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
+ const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<uint64_t, const FieldSchema*> table_col_pos_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ table_col_pos_to_field_schema_map[i] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnNameAccessPath>& name_access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (name_access_paths.empty()) return;
+
+ std::vector<TColumnNameAccessPath> paths;
+ bool has_top_level_only = false;
+ for (const auto& name_access_path : name_access_paths) {
+ DCHECK(name_access_path.path.size() >= 1);
+ TColumnNameAccessPath remaining_path;
+ if (name_access_path.path.size() > 1) {
+ remaining_path.path.assign(name_access_path.path.begin() + 1,
+ name_access_path.path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path.path = std::vector<std::string>();
+ }
+ if (remaining_path.path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ HiveParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+
+ // Find the field schema for this slot (may not exist for partition columns, etc.)
+ auto it = table_col_pos_to_field_schema_map.find(slot->col_pos());
+ if (it == table_col_pos_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_column_access_paths = slot->all_column_access_paths();
+
+ // primitive (non-nested) types: direct mapping by position
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // collect and process all_column_access_paths -> column_ids
+ if (all_column_access_paths.__isset.name_access_paths &&
+ !all_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(field_schema, all_column_access_paths.name_access_paths,
+ column_ids);
+ }
+
+ // collect and process predicate_column_access_paths -> filter_column_ids
+ const auto& predicate_column_access_paths = slot->predicate_column_access_paths();
+ if (predicate_column_access_paths.__isset.name_access_paths &&
+ !predicate_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_column_access_paths.name_access_paths,
+ filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/table/hive_reader.h b/be/src/vec/exec/format/table/hive_reader.h
index 6acd344e417..b9592c43243 100644
--- a/be/src/vec/exec/format/table/hive_reader.h
+++ b/be/src/vec/exec/format/table/hive_reader.h
@@ -66,6 +66,13 @@ public:
const RowDescriptor* row_descriptor,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts);
+
+private:
+ static ColumnIdResult _create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor);
+
+ static ColumnIdResult _create_column_ids_by_top_level_col_index(
+ const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor);
};
class HiveParquetReader final : public HiveReader {
@@ -88,6 +95,13 @@ public:
const std::unordered_map<std::string, int>* colname_to_slot_id,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts);
+
+private:
+ static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor);
+
+ static ColumnIdResult _create_column_ids_by_top_level_col_index(
+ const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor);
};
#include "common/compile_check_end.h"
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
new file mode 100644
index 00000000000..6d4e3726150
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "orc/Type.hh"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void IcebergOrcNestedColumnUtils::extract_nested_column_ids(
+ const orc::Type& type, const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<TColumnNameAccessPath>> child_paths_by_field_id;
+
+ for (const auto& access_path : paths) {
+ if (!access_path.path.empty()) {
+ std::string first_field_id = access_path.path[0];
+ TColumnNameAccessPath remaining;
+ if (access_path.path.size() > 1) {
+ remaining.path.assign(access_path.path.begin() + 1, access_path.path.end());
+ }
+ child_paths_by_field_id[first_field_id].push_back(std::move(remaining));
+ }
+ }
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (type.getKind() == orc::TypeKind::MAP) {
+ auto wildcard_it = child_paths_by_field_id.find("*");
+ if (wildcard_it != child_paths_by_field_id.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_field_id["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_field_id["KEYS"];
+ // Add an empty path to request full KEYS
+ TColumnNameAccessPath empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_field_id.erase(wildcard_it);
+ }
+ }
+
+ bool has_child_columns = false;
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) {
+ const orc::Type* child = type.getSubtype(i);
+
+ std::string child_field_id;
+ switch (type.getKind()) {
+ case orc::TypeKind::STRUCT:
+ if (!child->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) {
+ continue;
+ }
+ child_field_id = child->getAttributeValue(ICEBERG_ORC_ATTRIBUTE);
+ break;
+ case orc::TypeKind::LIST:
+ child_field_id = "*";
+ break;
+ case orc::TypeKind::MAP:
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_id = "KEYS";
+ } else if (i == 1) {
+ child_field_id = "VALUES";
+ }
+ break;
+ default:
+ child_field_id = "";
+ break;
+ }
+
+ if (child_field_id.empty() || child_field_id == "-1") {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_field_id.find(child_field_id);
+ if (child_paths_it != child_paths_by_field_id.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child = std::any_of(
+ child_paths.begin(), child_paths.end(),
+ [](const TColumnNameAccessPath& path) { return path.path.empty(); });
+
+ if (needs_full_child) {
+ uint64_t start_id = child->getColumnId();
+ uint64_t max_column_id = child->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(*child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(type.getColumnId());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h
new file mode 100644
index 00000000000..c01ab0465cd
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace orc {
+class Type;
+} // namespace orc
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class IcebergOrcNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const orc::Type& type,
+ const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids);
+
+private:
+ static constexpr const char* ICEBERG_ORC_ATTRIBUTE = "iceberg.id";
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
new file mode 100644
index 00000000000..6d3d47d8bac
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/parquet/schema_desc.h"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void IcebergParquetNestedColumnUtils::extract_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<TColumnNameAccessPath>> child_paths_by_field_id;
+
+ for (const auto& access_path : paths) {
+ if (!access_path.path.empty()) {
+ std::string first_field_id = access_path.path[0];
+ TColumnNameAccessPath remaining;
+ if (access_path.path.size() > 1) {
+ remaining.path.assign(access_path.path.begin() + 1, access_path.path.end());
+ }
+ child_paths_by_field_id[first_field_id].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP) {
+ auto wildcard_it = child_paths_by_field_id.find("*");
+ if (wildcard_it != child_paths_by_field_id.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_field_id["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_field_id["KEYS"];
+ // Add an empty path to request full KEYS
+ TColumnNameAccessPath empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_field_id.erase(wildcard_it);
+ }
+ }
+
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < field_schema.children.size(); ++i) {
+ const auto& child = field_schema.children[i];
+
+ std::string child_field_id;
+
+ bool is_list = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY;
+ bool is_map = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP;
+
+ if (is_list) {
+ child_field_id = "*";
+ } else if (is_map) {
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_id = "KEYS";
+ } else if (i == 1) {
+ child_field_id = "VALUES";
+ }
+
+ // Special handling for Parquet MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access =
+ child_paths_by_field_id.find("KEYS") != child_paths_by_field_id.end();
+ bool has_values_access =
+ child_paths_by_field_id.find("VALUES") != child_paths_by_field_id.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for RL/DL
+ if (!has_keys_access && has_values_access) {
+ // For map_values() queries, we need key's structure for correct RL/DL parsing.
+ // If key is a nested type (e.g., STRUCT), RL/DL info is stored at leaf columns.
+ // Add all column IDs from key's start to max (all leaves + intermediate nodes).
+ uint64_t key_start_id = child.get_column_id();
+ uint64_t key_max_id = child.get_max_column_id();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
+
+ } else {
+ child_field_id = std::to_string(child.field_id);
+ }
+
+ if (child_field_id.empty() || child_field_id == "-1") {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_field_id.find(child_field_id);
+ if (child_paths_it != child_paths_by_field_id.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child = std::any_of(child_paths.begin(), child_paths.end(),
+ [](const TColumnNameAccessPath& access_path) {
+ return access_path.path.empty();
+ });
+
+ if (needs_full_child) {
+ // Add all column IDs from current child node to max_column_id
+ // This efficiently handles all nested/complex cases in one loop
+ uint64_t start_id = child.get_column_id();
+ uint64_t max_column_id = child.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(field_schema.get_column_id());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h
new file mode 100644
index 00000000000..d0d252e4447
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class FieldDescriptor;
+struct FieldSchema;
+
+class IcebergParquetNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const FieldSchema& field_schema,
+ const std::vector<TColumnNameAccessPath>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index db6e44c3d4b..a9959599f84 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -17,6 +17,7 @@
#include "iceberg_reader.h"
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/Metrics_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <gen_cpp/parquet_types.h>
@@ -52,6 +53,8 @@
#include "vec/exec/format/orc/vorc_reader.h"
#include "vec/exec/format/parquet/schema_desc.h"
#include "vec/exec/format/parquet/vparquet_column_chunk_reader.h"
+#include "vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h"
+#include "vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include "vec/exec/format/table/table_format_reader.h"
namespace cctz {
@@ -295,7 +298,7 @@ IcebergTableReader::PositionDeleteRange IcebergTableReader::_get_range(
const ColumnDictI32& file_path_column) {
IcebergTableReader::PositionDeleteRange range;
size_t read_rows = file_path_column.get_data().size();
- const int* code_path = file_path_column.get_data().data();
+ const int* code_path = const_cast<int*>(file_path_column.get_data().data());
const int* code_path_start = code_path;
const int* code_path_end = code_path + read_rows;
while (code_path < code_path_end) {
@@ -445,18 +448,130 @@ Status IcebergParquetReader::init_reader(
}
_all_required_col_names = file_col_names;
+
+ auto column_id_result = _create_column_ids(field_desc, tuple_descriptor);
+ // const auto& file_col_names = column_id_result.column_names;
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
RETURN_IF_ERROR(init_row_filters());
- return parquet_reader->init_reader(_all_required_col_names, colname_to_value_range, conjuncts,
- tuple_descriptor, row_descriptor, colname_to_slot_id,
- not_single_slot_filter_conjuncts,
- slot_id_to_filter_conjuncts, table_info_node_ptr);
+ return parquet_reader->init_reader(
+ _all_required_col_names, colname_to_value_range, conjuncts, tuple_descriptor,
+ row_descriptor, colname_to_slot_id, not_single_slot_filter_conjuncts,
+ slot_id_to_filter_conjuncts, table_info_node_ptr, true, column_ids, filter_column_ids);
+}
+
+ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ // Create a mapping from iceberg field_id to FieldSchema for quick lookup
+ std::unordered_map<int, const FieldSchema*> iceberg_id_to_field_schema_map;
+
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ int iceberg_id = field_schema->field_id;
+ iceberg_id_to_field_schema_map[iceberg_id] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnNameAccessPath>& name_access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (name_access_paths.empty()) return;
+
+ std::vector<TColumnNameAccessPath> paths;
+ bool has_top_level_only = false;
+ for (const auto& name_access_path : name_access_paths) {
+ DCHECK(name_access_path.path.size() >= 1);
+ TColumnNameAccessPath remaining_path;
+ if (name_access_path.path.size() > 1) {
+ remaining_path.path.assign(name_access_path.path.begin() + 1,
+ name_access_path.path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path.path = std::vector<std::string>();
+ }
+ if (remaining_path.path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ IcebergParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths,
+ out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+ auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id());
+ if (it == iceberg_id_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_column_access_paths = slot->all_column_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_column_access_paths -> column_ids
+ if (all_column_access_paths.__isset.name_access_paths &&
+ !all_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(field_schema, all_column_access_paths.name_access_paths,
+ column_ids);
+ }
+
+ // collect and process predicate_column_access_paths -> filter_column_ids
+ const auto& predicate_column_access_paths = slot->predicate_column_access_paths();
+ if (predicate_column_access_paths.__isset.name_access_paths &&
+ !predicate_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_column_access_paths.name_access_paths,
+ filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status IcebergParquetReader ::_read_position_delete_file(const TFileRangeDesc* delete_range,
DeleteFile* position_delete) {
- ParquetReader parquet_delete_reader(_profile, _params, *delete_range,
- READ_DELETE_FILE_BATCH_SIZE, &_state->timezone_obj(),
- _io_ctx, _state, _meta_cache);
+ ParquetReader parquet_delete_reader(
+ _profile, _params, *delete_range, READ_DELETE_FILE_BATCH_SIZE,
+ const_cast<cctz::time_zone*>(&_state->timezone_obj()), _io_ctx, _state, _meta_cache);
RETURN_IF_ERROR(parquet_delete_reader.init_reader(
delete_file_col_names, nullptr, {}, nullptr, nullptr, nullptr, nullptr, nullptr,
TableSchemaChangeHelper::ConstNode::get_instance(), false));
@@ -527,11 +642,113 @@ Status IcebergOrcReader::init_reader(
}
}
+ auto column_id_result = _create_column_ids(orc_type_ptr, tuple_descriptor);
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
RETURN_IF_ERROR(init_row_filters());
return orc_reader->init_reader(&_all_required_col_names, colname_to_value_range, conjuncts,
false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts,
- table_info_node_ptr);
+ table_info_node_ptr, column_ids, filter_column_ids);
+}
+
+ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column name (lower-cased) -> orc::Type*
+ std::unordered_map<int, const orc::Type*> iceberg_id_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ if (!orc_sub_type->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) {
+ continue;
+ }
+ int iceberg_id = std::stoi(orc_sub_type->getAttributeValue(ICEBERG_ORC_ATTRIBUTE));
+ iceberg_id_to_orc_type_map[iceberg_id] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnNameAccessPath>& name_access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (name_access_paths.empty()) return;
+
+ std::vector<TColumnNameAccessPath> paths;
+ bool has_top_level_only = false;
+ for (const auto& name_access_path : name_access_paths) {
+ DCHECK(name_access_path.path.size() >= 1);
+ TColumnNameAccessPath remaining_path;
+ if (name_access_path.path.size() > 1) {
+ remaining_path.path.assign(name_access_path.path.begin() + 1,
+ name_access_path.path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path.path = std::vector<std::string>();
+ }
+ if (remaining_path.path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ IcebergOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = iceberg_id_to_orc_type_map.find(slot->col_unique_id());
+ if (it == iceberg_id_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_column_access_paths = slot->all_column_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // nested types:
+
+ // collect and process all_column_access_paths -> column_ids
+ if (all_column_access_paths.__isset.name_access_paths &&
+ !all_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(orc_field, all_column_access_paths.name_access_paths, column_ids);
+ }
+
+ // collect and process predicate_column_access_paths -> filter_column_ids
+ const auto& predicate_column_access_paths = slot->predicate_column_access_paths();
+ if (predicate_column_access_paths.__isset.name_access_paths &&
+ !predicate_column_access_paths.name_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_column_access_paths.name_access_paths,
+ filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete_range,
diff --git a/be/src/vec/exec/format/table/iceberg_reader.h b/be/src/vec/exec/format/table/iceberg_reader.h
index 6b0e3fe4b3e..d123261714e 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.h
+++ b/be/src/vec/exec/format/table/iceberg_reader.h
@@ -187,6 +187,9 @@ protected:
}
private:
+ static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor);
+
Status _read_position_delete_file(const TFileRangeDesc* delete_range,
DeleteFile* position_delete) final;
};
@@ -226,6 +229,10 @@ protected:
_meta_cache);
}
+private:
+ static ColumnIdResult _create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor);
+
private:
static const std::string ICEBERG_ORC_ATTRIBUTE;
};
diff --git a/be/src/vec/exec/format/table/table_format_reader.cpp b/be/src/vec/exec/format/table/table_format_reader.cpp
index b3b7e1df336..2c614777a89 100644
--- a/be/src/vec/exec/format/table/table_format_reader.cpp
+++ b/be/src/vec/exec/format/table/table_format_reader.cpp
@@ -76,8 +76,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
if (file_field.data_type->get_primitive_type() != TYPE_MAP) [[unlikely]] {
return SCHEMA_ERROR;
}
- MOCK_REMOVE(DCHECK(file_field.children.size() == 1));
- MOCK_REMOVE(DCHECK(file_field.children[0].children.size() == 2));
+ MOCK_REMOVE(DCHECK(file_field.children.size() == 2));
std::shared_ptr<TableSchemaChangeHelper::Node> key_node = nullptr;
{
@@ -85,8 +84,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
assert_cast<const DataTypeMap*>(remove_nullable(table_data_type).get())
->get_key_type());
- RETURN_IF_ERROR(
- by_parquet_name(key_type, file_field.children[0].children[0], key_node));
+ RETURN_IF_ERROR(by_parquet_name(key_type, file_field.children[0], key_node));
}
std::shared_ptr<TableSchemaChangeHelper::Node> value_node = nullptr;
@@ -95,8 +93,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
assert_cast<const DataTypeMap*>(remove_nullable(table_data_type).get())
->get_value_type());
- RETURN_IF_ERROR(
- by_parquet_name(value_type, file_field.children[0].children[1], value_node));
+ RETURN_IF_ERROR(by_parquet_name(value_type, file_field.children[1], value_node));
}
node = std::make_shared<TableSchemaChangeHelper::MapNode>(key_node, value_node);
break;
@@ -430,19 +427,17 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id(
MOCK_REMOVE(DCHECK(table_schema.nestedField.map_field.key_field.field_ptr != nullptr));
MOCK_REMOVE(DCHECK(table_schema.nestedField.map_field.value_field.field_ptr != nullptr));
- MOCK_REMOVE(DCHECK(parquet_field.children.size() == 1));
- MOCK_REMOVE(DCHECK(parquet_field.children[0].children.size() == 2));
+ MOCK_REMOVE(DCHECK(parquet_field.children.size() == 2));
std::shared_ptr<TableSchemaChangeHelper::Node> key_node = nullptr;
std::shared_ptr<TableSchemaChangeHelper::Node> value_node = nullptr;
RETURN_IF_ERROR(by_parquet_field_id(*table_schema.nestedField.map_field.key_field.field_ptr,
- parquet_field.children[0].children[0], key_node,
- exist_field_id));
+ parquet_field.children[0], key_node, exist_field_id));
- RETURN_IF_ERROR(by_parquet_field_id(
- *table_schema.nestedField.map_field.value_field.field_ptr,
- parquet_field.children[0].children[1], value_node, exist_field_id));
+ RETURN_IF_ERROR(
+ by_parquet_field_id(*table_schema.nestedField.map_field.value_field.field_ptr,
+ parquet_field.children[1], value_node, exist_field_id));
node = std::make_shared<TableSchemaChangeHelper::MapNode>(key_node, value_node);
break;
diff --git a/be/src/vec/exec/format/table/table_format_reader.h b/be/src/vec/exec/format/table/table_format_reader.h
index 1f2a41af930..d33f5bc9cf3 100644
--- a/be/src/vec/exec/format/table/table_format_reader.h
+++ b/be/src/vec/exec/format/table/table_format_reader.h
@@ -388,5 +388,16 @@ public:
};
};
+struct ColumnIdResult {
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ ColumnIdResult() = default; // Add default constructor
+
+ ColumnIdResult(std::set<uint64_t> column_ids_, std::set<uint64_t> filter_column_ids_)
+ : column_ids(std::move(column_ids_)),
+ filter_column_ids(std::move(filter_column_ids_)) {}
+};
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp
index c0b440fa6f3..1ca195c319e 100644
--- a/be/src/vec/exec/scan/file_scanner.cpp
+++ b/be/src/vec/exec/scan/file_scanner.cpp
@@ -335,7 +335,7 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al
return Status::OK();
}
-Status FileScanner::_process_conjuncts_for_dict_filter() {
+Status FileScanner::_process_conjuncts() {
_slot_id_to_filter_conjuncts.clear();
_not_single_slot_filter_conjuncts.clear();
for (auto& conjunct : _push_down_conjuncts) {
@@ -373,7 +373,7 @@ Status FileScanner::_process_late_arrival_conjuncts() {
for (size_t i = 0; i != _conjuncts.size(); ++i) {
RETURN_IF_ERROR(_conjuncts[i]->clone(_state, _push_down_conjuncts[i]));
}
- RETURN_IF_ERROR(_process_conjuncts_for_dict_filter());
+ RETURN_IF_ERROR(_process_conjuncts());
_discard_conjuncts();
}
if (_applied_rf_num == _total_rf_num) {
@@ -386,6 +386,8 @@ void FileScanner::_get_slot_ids(VExpr* expr, std::vector<int>* slot_ids) {
for (auto& child_expr : expr->children()) {
if (child_expr->is_slot_ref()) {
VSlotRef* slot_ref = reinterpret_cast<VSlotRef*>(child_expr.get());
+ SlotDescriptor* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id());
+ slot_desc->set_is_predicate(true);
slot_ids->emplace_back(slot_ref->slot_id());
} else {
_get_slot_ids(child_expr.get(), slot_ids);
diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h
index 1a44d06c77b..8c4682ca89f 100644
--- a/be/src/vec/exec/scan/file_scanner.h
+++ b/be/src/vec/exec/scan/file_scanner.h
@@ -252,7 +252,7 @@ private:
void _init_runtime_filter_partition_prune_ctxs();
void _init_runtime_filter_partition_prune_block();
Status _process_runtime_filters_partition_prune(bool& is_partition_pruned);
- Status _process_conjuncts_for_dict_filter();
+ Status _process_conjuncts();
Status _process_late_arrival_conjuncts();
void _get_slot_ids(VExpr* expr, std::vector<int>* slot_ids);
Status _generate_truncate_columns(bool need_to_get_parsed_schema);
diff --git a/be/src/vec/exec/scan/olap_scanner.cpp b/be/src/vec/exec/scan/olap_scanner.cpp
index 514f6dd8f12..316c0083f1e 100644
--- a/be/src/vec/exec/scan/olap_scanner.cpp
+++ b/be/src/vec/exec/scan/olap_scanner.cpp
@@ -546,14 +546,14 @@ Status OlapScanner::_init_return_columns() {
}
const auto& column = tablet_schema->column(index);
- if (!slot->all_access_paths().name_access_paths.empty()) {
+ if (!slot->all_column_access_paths().name_access_paths.empty()) {
_tablet_reader_params.all_access_paths.insert(
- {column.unique_id(), slot->all_access_paths()});
+ {column.unique_id(), slot->all_column_access_paths()});
}
- if (!slot->predicate_access_paths().name_access_paths.empty()) {
+ if (!slot->predicate_column_access_paths().name_access_paths.empty()) {
_tablet_reader_params.predicate_access_paths.insert(
- {column.unique_id(), slot->predicate_access_paths()});
+ {column.unique_id(), slot->predicate_column_access_paths()});
}
_return_columns.push_back(index);
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
new file mode 100644
index 00000000000..832aaef64a0
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
new file mode 100644
index 00000000000..36f589e393e
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json
new file mode 100644
index 00000000000..97208990039
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json
@@ -0,0 +1,145 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "daf54708-3539-4de4-a130-322cd4c3fdc4",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1758865887090,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zlib"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json
new file mode 100644
index 00000000000..12ec38bbdcd
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json
@@ -0,0 +1,176 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "daf54708-3539-4de4-a130-322cd4c3fdc4",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1758865954986,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zlib"
+ },
+ "current-snapshot-id" : 2197982135324349212,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 2197982135324349212,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 2197982135324349212,
+ "timestamp-ms" : 1758865954986,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0674",
+ "added-data-files" : "2",
+ "added-records" : "10",
+ "added-files-size" : "4611",
+ "changed-partition-count" : "1",
+ "total-records" : "10",
+ "total-files-size" : "4611",
+ "total-data-files" : "2",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1758865954986,
+ "snapshot-id" : 2197982135324349212
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1758865887090,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro
new file mode 100644
index 00000000000..1887220ebf0
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro
new file mode 100644
index 00000000000..7218d1326d3
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..160d000fed6
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..3ec61d58469
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..b0f56644ae5
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..dee411d7ef7
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..4264e841218
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..0264972b11d
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..3db75d8c01a
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..4838bb7d6e7
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json
new file mode 100644
index 00000000000..e6fc794e81f
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json
@@ -0,0 +1,145 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "baf92432-1443-47fd-971a-0a9df2432e4a",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1758188914000,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json
new file mode 100644
index 00000000000..d52c580cbf2
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json
@@ -0,0 +1,176 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "baf92432-1443-47fd-971a-0a9df2432e4a",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1758188959726,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 1689809712605583552,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 1689809712605583552,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 1689809712605583552,
+ "timestamp-ms" : 1758188959726,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0668",
+ "added-data-files" : "8",
+ "added-records" : "10",
+ "added-files-size" : "32551",
+ "changed-partition-count" : "1",
+ "total-records" : "10",
+ "total-files-size" : "32551",
+ "total-data-files" : "8",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1758188959726,
+ "snapshot-id" : 1689809712605583552
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1758188914000,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro
new file mode 100644
index 00000000000..201c7908bc4
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro
new file mode 100644
index 00000000000..1f6548d7ba8
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc
new file mode 100644
index 00000000000..2f63195ff34
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json
new file mode 100644
index 00000000000..174f7a92c79
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json
@@ -0,0 +1,548 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "a93bcb6d-568f-4fc2-8235-dc8e218c1beb",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1760884376565,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json
new file mode 100644
index 00000000000..3fc692fea56
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json
@@ -0,0 +1,579 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "a93bcb6d-568f-4fc2-8235-dc8e218c1beb",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1760884424603,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 514179801967160228,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 514179801967160228,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 514179801967160228,
+ "timestamp-ms" : 1760884424603,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0681",
+ "added-data-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "7181",
+ "changed-partition-count" : "1",
+ "total-records" : "1",
+ "total-files-size" : "7181",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1760884424603,
+ "snapshot-id" : 514179801967160228
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1760884376565,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro
new file mode 100644
index 00000000000..d8b578a8f86
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro
@@ -0,0 +1 @@
+Obj schema�]{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"profile","required":false,"type":{"type":"struct","fields":[{"id":9,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":12,"name":"street","required":false,"type":"string"},{"id":13,"name":"city","required":false,"type":"string"},{"id":14,"name":"coordinates","required":false,"type":{"type":"struc [...]
format-version2"partition-spec-id0 iceberg.schema�+{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position delet [...]
partition-spec[]contentdata v�7�l���r���P�Kq=��l��XY~,� ð,�5;3� *baia��6_�kێͮ"!!!�co��{�B=��l��}8�s� 8̇�����m7 `��bg�4{dx��N7�̕N ��j`�9Q�cmz��A�[�]� s��{��r.6[A���7�n���3}��Vo�����7\3�S6�5�j�V}�5�Vq+ިeX=i٥�;y~�o/��H�I�"-я ��s� T�I��GA�A �J�* �,��1��'�8&�����}�:�\��.箨���������"X��d[,OD��\����ܐw7 ��p���C�#J?&����H�"���/(�M�%�^��!�^����[��H{G�}ʾ'��>�~@\dNp��8łӬp?gXr�s<�*k�s� ��gEqHA *@9�@����\�a���ul��������봻9����ѵ���.�a0 �r����P��6��Q��T*ò-;+��am�i���ݜ2�iW�親,�=<rbnr�_��O s2�v�7�
�l���r�
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro
new file mode 100644
index 00000000000..7621edf3b7f
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet
new file mode 100644
index 00000000000..014ad377122
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json
new file mode 100644
index 00000000000..8bf743851ff
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json
@@ -0,0 +1,548 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "d3b72ae1-4880-4c3e-9111-ca1af3cc9c7c",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1760884395135,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json
new file mode 100644
index 00000000000..de90e000742
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json
@@ -0,0 +1,579 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "d3b72ae1-4880-4c3e-9111-ca1af3cc9c7c",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1760884432817,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 6622777947381868002,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 6622777947381868002,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 6622777947381868002,
+ "timestamp-ms" : 1760884432817,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0681",
+ "added-data-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "21586",
+ "changed-partition-count" : "1",
+ "total-records" : "1",
+ "total-files-size" : "21586",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1760884432817,
+ "snapshot-id" : 6622777947381868002
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1760884395135,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro
new file mode 100644
index 00000000000..6071b684120
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro
@@ -0,0 +1,5 @@
+Obj schema�]{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"profile","required":false,"type":{"type":"struct","fields":[{"id":9,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":12,"name":"street","required":false,"type":"string"},{"id":13,"name":"city","required":false,"type":"string"},{"id":14,"name":"coordinates","required":false,"type":{"type":"struc [...]
format-version2"partition-spec-id0 iceberg.schema�+{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position delet [...]
partition-spec[]contentdata ��b���}zd�ɤb����
+Œ�kaǟ���Lr���! AB�^ri��H�g��D*�?bC�K�br���rM%� �A�C)����!H)����M�:��(����{����wx����~����{�����߾�F xIe�T� :�X4V�fW�,�+ -��XZ���`q���v@-���Լ3�7�Z�bh�|��4�n�T��U����%Gb\T.iRh,R�H,$�%-,��HTT�qE��x�U$љN gg���Z��@��D:�ri�1\�䄜����K#] R;�G�Xaܘl'�qʚ��픑62F�X�
+��C�4_��m�Qm�-���`���:>� �`�M�na���m� ��a�]l�ak �|������lGn& �% �i� p�(�h�&�7EӔ��)CYZ����.�eS��Ԣ6u��H��m �C�D�B�#m# �v�v����������߁7 ����0� Q��8LB�`���4d �P 0�Є����EXG�@�!l"l!��;�{��GlShH`,�q� �泌�a� �;��㓵rUHԴ �+�
+�� 3J�*����MmMX�Y��ȗ �H�ٓ[�DŽ���x�ʝ�,�??� ν�
+:�Ԛ&k���(N�)��] ��H$"J!)d�> _�N�� 7x�3�������E�uR�����s������O�q~�_�����b���}zd�ɤb���
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro
new file mode 100644
index 00000000000..a3eee4e6a11
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_orc/_SUCCESS b/be/test/exec/test_data/nested_user_profiles_orc/_SUCCESS
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc b/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc
new file mode 100644
index 00000000000..8a0f8bc17b4
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc differ
diff --git a/be/test/exec/test_data/nested_user_profiles_parquet/_SUCCESS b/be/test/exec/test_data/nested_user_profiles_parquet/_SUCCESS
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet b/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet
new file mode 100644
index 00000000000..73ae0061c21
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet differ
diff --git a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
index 797ab9fc7db..edd131a2410 100644
--- a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
+++ b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
@@ -140,20 +140,18 @@ TEST_F(ParquetThriftReaderTest, complex_nested_file) {
ASSERT_EQ(schemaDescriptor.get_column_index("hobby"), 2);
auto hobby = schemaDescriptor.get_column("hobby");
- // should be parsed as ARRAY<MAP<STRUCT<STRING,STRING>>>
- ASSERT_TRUE(hobby->children.size() == 1 && hobby->children[0].children.size() == 1 &&
- hobby->children[0].children[0].children.size() == 2);
+ // should be parsed as ARRAY<MAP<KEY,VALUE>>
+ ASSERT_TRUE(hobby->children.size() == 1 && hobby->children[0].children.size() == 2);
ASSERT_TRUE(hobby->data_type->get_primitive_type() == TYPE_ARRAY &&
- hobby->children[0].data_type->get_primitive_type() == TYPE_MAP &&
- hobby->children[0].children[0].data_type->get_primitive_type() == TYPE_STRUCT);
+ hobby->children[0].data_type->get_primitive_type() == TYPE_MAP);
// hobby(opt) --- bag(rep) --- array_element(opt) --- map(rep)
// \------- key(req)
// \------- value(opt)
// R=0,D=1 R=1,D=2 R=1,D=3 R=2,D=4
// \------ R=2,D=4
// \------ R=2,D=5
- auto h_key = hobby->children[0].children[0].children[0];
- auto h_value = hobby->children[0].children[0].children[1];
+ auto h_key = hobby->children[0];
+ auto h_value = hobby->children[1];
ASSERT_TRUE(h_key.repetition_level == 2 && h_key.definition_level == 4);
ASSERT_TRUE(h_value.repetition_level == 2 && h_value.definition_level == 5);
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
new file mode 100644
index 00000000000..ad881994acd
--- /dev/null
+++ b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
@@ -0,0 +1,1183 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+#include "vec/exec/format/table/hive_reader.h"
+
+namespace doris::vectorized {
+
+// Define the column access path configuration structure
+struct ColumnAccessPathConfig {
+ std::string column_name;
+ std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
+ std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
+ bool use_name_paths = true; // true=use field names, false=use indices
+};
+
+// ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+// 0: struct (table/root)
+// 1: id (bigint)
+// 2: name (string)
+// 3: profile (struct)
+// 4: address (struct)
+// 5: street (string)
+// 6: city (string)
+// 7: coordinates (struct)
+// 8: lat (double)
+// 9: lng (double)
+// 10: contact (struct)
+// 11: email (string)
+// 12: phone (struct)
+// 13: country_code (string)
+// 14: number (string)
+// 15: hobbies (array)
+// 16: element (struct)
+// 17: name (string)
+// 18: level (int)
+// 19: tags (array)
+// 20: element (string)
+// 21: friends (array)
+// 22: element (struct)
+// 23: user_id (bigint)
+// 24: nickname (string)
+// 25: friendship_level (int)
+// 26: recent_activity (array)
+// 27: element (struct)
+// 28: action (string)
+// 29: details (array)
+// 30: element (struct)
+// 31: key (string)
+// 32: value (string)
+// 33: attributes (map)
+// 34: key (string)
+// 35: value (string)
+// 36: complex_attributes (map)
+// 37: key (string)
+// 38: value (struct)
+// 39: metadata (struct)
+// 40: version (string)
+// 41: created_time (timestamp)
+// 42: last_updated (timestamp)
+// 43: quality_score (double)
+// 44: historical_scores (array)
+// 45: element (struct)
+// 46: period (string)
+// 47: score (double)
+// 48: components (array)
+// 49: element (struct)
+// 50: component_name (string)
+// 51: weight (float)
+// 52: sub_scores (map)
+// 53: key (string)
+// 54: value (double)
+// 55: trends (struct)
+// 56: direction (string)
+// 57: magnitude (double)
+// 58: confidence_interval (struct)
+// 59: lower (double)
+// 60: upper (double)
+// 61: hierarchical_data (map)
+// 62: key (string)
+// 63: value (struct)
+// 64: category (string)
+// 65: sub_items (array)
+// 66: element (struct)
+// 67: item_id (bigint)
+// 68: properties (map)
+// 69: key (string)
+// 70: value (string)
+// 71: metrics (struct)
+// 72: value (double)
+// 73: unit (string)
+// 74: summary (struct)
+// 75: total_count (int)
+// 76: average_value (double)
+// 77: validation_rules (struct)
+// 78: required (boolean)
+// 79: constraints (array)
+// 80: element (struct)
+// 81: rule_type (string)
+// 82: parameters (map)
+// 83: key (string)
+// 84: value (string)
+// 85: error_message (string)
+// 86: complex_constraint (struct)
+// 87: logic_operator (string)
+// 88: operands (array)
+// 89: element (struct)
+// 90: field_path (string)
+// 91: operator (string)
+// 92: comparison_value (string)
+
+class HiveReaderCreateColumnIdsTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types,
+ const std::vector<ColumnAccessPathConfig>& access_configs = {}) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [this, &access_configs](
+ TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::HIVE_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "id") {
+ // id: bigint
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::BIGINT);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "name") {
+ // name: string
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::STRING);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "tags") {
+ // tags: array<string>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_node;
+ element_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType element_scalar;
+ element_scalar.__set_type(TPrimitiveType::STRING);
+ element_node.__set_scalar_type(element_scalar);
+ type.types.push_back(element_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "friends") {
+ // friends: array<struct<user_id:bigint, nickname:string, friendship_level:int>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField user_id_field;
+ user_id_field.__set_name("user_id");
+ user_id_field.__set_contains_null(true);
+ element_fields.push_back(user_id_field);
+ TStructField nickname_field;
+ nickname_field.__set_name("nickname");
+ nickname_field.__set_contains_null(true);
+ element_fields.push_back(nickname_field);
+ TStructField friendship_level_field;
+ friendship_level_field.__set_name("friendship_level");
+ friendship_level_field.__set_contains_null(true);
+ element_fields.push_back(friendship_level_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // user_id: bigint
+ TTypeNode user_id_node;
+ user_id_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType user_id_scalar;
+ user_id_scalar.__set_type(TPrimitiveType::BIGINT);
+ user_id_node.__set_scalar_type(user_id_scalar);
+ type.types.push_back(user_id_node);
+ // nickname: string
+ TTypeNode nickname_node;
+ nickname_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType nickname_scalar;
+ nickname_scalar.__set_type(TPrimitiveType::STRING);
+ nickname_node.__set_scalar_type(nickname_scalar);
+ type.types.push_back(nickname_node);
+ // friendship_level: int
+ TTypeNode friendship_level_node;
+ friendship_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType friendship_level_scalar;
+ friendship_level_scalar.__set_type(TPrimitiveType::INT);
+ friendship_level_node.__set_scalar_type(friendship_level_scalar);
+ type.types.push_back(friendship_level_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "recent_activity") {
+ // recent_activity: array<struct<action:string, details:array<struct<key:string, value:string>>>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField action_field;
+ action_field.__set_name("action");
+ action_field.__set_contains_null(true);
+ element_fields.push_back(action_field);
+ TStructField details_field;
+ details_field.__set_name("details");
+ details_field.__set_contains_null(true);
+ element_fields.push_back(details_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // action: string
+ TTypeNode action_node;
+ action_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType action_scalar;
+ action_scalar.__set_type(TPrimitiveType::STRING);
+ action_node.__set_scalar_type(action_scalar);
+ type.types.push_back(action_node);
+ // details: array<struct<key:string, value:string>>
+ TTypeNode details_array_node;
+ details_array_node.__set_type(TTypeNodeType::ARRAY);
+ details_array_node.__set_contains_nulls({true});
+ type.types.push_back(details_array_node);
+ TTypeNode details_element_struct_node;
+ details_element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> details_element_fields;
+ TStructField key_field;
+ key_field.__set_name("key");
+ key_field.__set_contains_null(true);
+ details_element_fields.push_back(key_field);
+ TStructField value_field;
+ value_field.__set_name("value");
+ value_field.__set_contains_null(true);
+ details_element_fields.push_back(value_field);
+ details_element_struct_node.__set_struct_fields(details_element_fields);
+ type.types.push_back(details_element_struct_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "attributes") {
+ // attributes: map<string, string>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "complex_attributes") {
+ // complex_attributes: map<string, struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>
+ TTypeNode value_struct_node;
+ value_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> value_fields;
+ TStructField metadata_field;
+ metadata_field.__set_name("metadata");
+ metadata_field.__set_contains_null(true);
+ value_fields.push_back(metadata_field);
+ value_struct_node.__set_struct_fields(value_fields);
+ type.types.push_back(value_struct_node);
+ // metadata: struct<version:string, created_time:timestamp, last_updated:timestamp>
+ TTypeNode metadata_struct_node;
+ metadata_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> metadata_fields;
+ TStructField version_field;
+ version_field.__set_name("version");
+ version_field.__set_contains_null(true);
+ metadata_fields.push_back(version_field);
+ TStructField created_time_field;
+ created_time_field.__set_name("created_time");
+ created_time_field.__set_contains_null(true);
+ metadata_fields.push_back(created_time_field);
+ TStructField last_updated_field;
+ last_updated_field.__set_name("last_updated");
+ last_updated_field.__set_contains_null(true);
+ metadata_fields.push_back(last_updated_field);
+ metadata_struct_node.__set_struct_fields(metadata_fields);
+ type.types.push_back(metadata_struct_node);
+ // version: string
+ TTypeNode version_node;
+ version_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType version_scalar;
+ version_scalar.__set_type(TPrimitiveType::STRING);
+ version_node.__set_scalar_type(version_scalar);
+ type.types.push_back(version_node);
+ // created_time: timestamp
+ TTypeNode created_time_node;
+ created_time_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType created_time_scalar;
+ created_time_scalar.__set_type(TPrimitiveType::DATETIME);
+ created_time_node.__set_scalar_type(created_time_scalar);
+ type.types.push_back(created_time_node);
+ // last_updated: timestamp
+ TTypeNode last_updated_node;
+ last_updated_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType last_updated_scalar;
+ last_updated_scalar.__set_type(TPrimitiveType::DATETIME);
+ last_updated_node.__set_scalar_type(last_updated_scalar);
+ type.types.push_back(last_updated_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY nodes need contains_nulls, SCALAR nodes do not
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // Regular types
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+
+ // Use configuration to set column_access_paths
+ for (const auto& config : access_configs) {
+ set_column_access_paths(tslot_desc, config);
+ }
+
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function: Set column access paths
+ void set_column_access_paths(TSlotDescriptor& tslot_desc,
+ const ColumnAccessPathConfig& config) {
+ if (config.column_name != tslot_desc.colName) {
+ return; // Not the target column, skip
+ }
+
+ // Set all_column_access_paths
+ if (!config.all_column_paths.empty()) {
+ TColumnAccessPaths access_paths;
+ access_paths.__set_type(doris::TAccessPathType::NAME);
+ std::vector<TColumnNameAccessPath> paths;
+
+ for (const auto& path_vector : config.all_column_paths) {
+ TColumnNameAccessPath path;
+ path.__set_path(path_vector);
+ paths.push_back(path);
+ }
+
+ access_paths.__set_name_access_paths(paths);
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+
+ // Set predicate_column_access_paths
+ if (!config.predicate_paths.empty()) {
+ TColumnAccessPaths access_paths;
+ access_paths.__set_type(doris::TAccessPathType::NAME);
+ std::vector<TColumnNameAccessPath> paths;
+
+ for (const auto& path_vector : config.predicate_paths) {
+ TColumnNameAccessPath path;
+ path.__set_path(path_vector);
+ paths.push_back(path);
+ }
+
+ access_paths.__set_name_access_paths(paths);
+ tslot_desc.__set_predicate_access_paths(access_paths);
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+
+ // Helper function: Create and setup ParquetReader
+ std::tuple<std::unique_ptr<HiveParquetReader>, const FieldDescriptor*> create_parquet_reader(
+ const std::string& test_file) {
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ RuntimeState runtime_state((TQueryGlobals()));
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size();
+ scan_range.path = test_file;
+ RuntimeProfile profile("test_profile");
+
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+ auto generic_reader =
+ ParquetReader::create_unique(&profile, scan_params, scan_range, 1024, &ctz, nullptr,
+ &runtime_state, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ const FieldDescriptor* field_desc = nullptr;
+ st = parquet_reader->get_file_metadata_schema(&field_desc);
+ if (!st.ok() || !field_desc) {
+ return {nullptr, nullptr};
+ }
+
+ auto hive_reader = std::make_unique<HiveParquetReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(hive_reader), field_desc};
+ }
+
+ // Helper function: Create and setup OrcReader
+ std::tuple<std::unique_ptr<HiveOrcReader>, const orc::Type*> create_orc_reader(
+ const std::string& test_file) {
+ // Open the Hive Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ OrcReader::create_unique(&profile, &runtime_state, scan_params, scan_range, 1024,
+ "CST", nullptr, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto orc_reader = static_cast<OrcReader*>(generic_reader.get());
+ // Get FieldDescriptor from Orc file
+ const orc::Type* orc_type_ptr = nullptr;
+ st = orc_reader->get_file_type(&orc_type_ptr);
+ if (!st.ok() || !orc_type_ptr) {
+ return {nullptr, nullptr};
+ }
+
+ // Create HiveOrcReader
+ auto hive_reader = std::make_unique<HiveOrcReader>(std::move(generic_reader), &profile,
+ &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(hive_reader), orc_type_ptr};
+ }
+
+ // Helper function: Run Parquet test with different column ID extraction methods
+ void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_parquet/"
+ "part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet";
+
+ auto [hive_reader, field_desc] = create_parquet_reader(test_file);
+ if (!hive_reader || !field_desc) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ actual_result = HiveParquetReader::_create_column_ids_by_top_level_col_index(
+ field_desc, tuple_descriptor);
+ } else {
+ actual_result = HiveParquetReader::_create_column_ids(field_desc, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+
+ // Helper function: Run ORC test with different column ID extraction methods
+ void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_orc/"
+ "part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc";
+
+ auto [hive_reader, orc_type] = create_orc_reader(test_file);
+ if (!hive_reader || !orc_type) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ actual_result = HiveOrcReader::_create_column_ids_by_top_level_col_index(
+ orc_type, tuple_descriptor);
+ } else {
+ actual_result = HiveOrcReader::_create_column_ids(orc_type, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+};
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_1) {
+ // Properties:
+
+ // Configure access paths for profile column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "address", "coordinates", "lng"},
+ {"profile", "contact", "email"},
+ {"profile", "hobbies", "*", "level"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_2) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_3) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_4) {
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {};
+ access_config.predicate_paths = {};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // 19: tags (array)
+ // 20: element (string)
+ // 21: friends (array)
+ // 22: element (struct)
+ // 23: user_id (bigint)
+ // 24: nickname (string)
+ // 25: friendship_level (int)
+ // 26: recent_activity (array)
+ // 27: element (struct)
+ // 28: action (string)
+ // 29: details (array)
+ // 30: element (struct)
+ // 31: key (string)
+ // 32: value (string)
+
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // Configure access paths for friends column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "friends";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"friends", "*", "nickname"},
+ {"friends", "*", "friendship_level"}};
+ access_config.predicate_paths = {
+ {"friends", "*", "nickname"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // Configure access paths for recent_activity column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "recent_activity";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"recent_activity", "*", "action"},
+ {"recent_activity", "*", "details", "*", "value"}};
+ access_config.predicate_paths = {
+ {"recent_activity", "*", "action"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids, true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // 33: attributes (map)
+ // 34: key (string)
+ // 35: value (string)
+ // 36: complex_attributes (map)
+ // 37: key (string)
+ // 38: value (struct)
+ // 39: metadata (struct)
+ // 40: version (string)
+ // 41: created_time (timestamp)
+ // 42: last_updated (timestamp)
+ // 43: quality_score (double)
+ // 44: historical_scores (array)
+ // 45: element (struct)
+ // 46: period (string)
+ // 47: score (double)
+ // 48: components (array)
+ // 49: element (struct)
+ // 50: component_name (string)
+ // 51: weight (float)
+ // 52: sub_scores (map)
+ // 53: key (string)
+ // 54: value (double)
+ // 55: trends (struct)
+ // 56: direction (string)
+ // 57: magnitude (double)
+ // 58: confidence_interval (struct)
+ // 59: lower (double)
+ // 60: upper (double)
+ // 61: hierarchical_data (map)
+ // 62: key (string)
+ // 63: value (struct)
+ // 64: category (string)
+ // 65: sub_items (array)
+ // 66: element (struct)
+ // 67: item_id (bigint)
+ // 68: properties (map)
+ // 69: key (string)
+ // 70: value (string)
+ // 71: metrics (struct)
+ // 72: value (double)
+ // 73: unit (string)
+ // 74: summary (struct)
+ // 75: total_count (int)
+ // 76: average_value (double)
+ // 77: validation_rules (struct)
+ // 78: required (boolean)
+ // 79: constraints (array)
+ // 80: element (struct)
+ // 81: rule_type (string)
+ // 82: parameters (map)
+ // 83: key (string)
+ // 84: value (string)
+ // 85: error_message (string)
+ // 86: complex_constraint (struct)
+ // 87: logic_operator (string)
+ // 88: operands (array)
+ // 89: element (struct)
+ // 90: field_path (string)
+ // 91: operator (string)
+ // 92: comparison_value (string)
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // Configure access paths for complex_attributes column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "complex_attributes";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {
+ {"complex_attributes", "*", "metadata", "version"},
+ {"complex_attributes", "*", "historical_scores", "*", "components", "*",
+ "sub_scores"},
+ {"complex_attributes", "*", "hierarchical_data", "VALUES"},
+ {"complex_attributes", "VALUES", "validation_rules", "constraints", "*",
+ "parameters", "KEYS"}};
+ access_config.predicate_paths = {{"complex_attributes", "*", "metadata", "version"}
+
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // parquet values should access keys
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> parquet_expected_column_ids = {
+ 2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53, 54, 61, 62, 63, 64,
+ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> parquet_expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
+ parquet_expected_filter_column_ids);
+ run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
+ parquet_expected_filter_column_ids, true);
+ }
+
+ {
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> orc_expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
+ 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
+ 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> orc_expected_filter_column_ids = {36, 37, 38, 39, 40};
+ run_orc_test_with_method(access_configs, orc_expected_column_ids,
+ orc_expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, orc_expected_column_ids,
+ orc_expected_filter_column_ids, true);
+ }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
new file mode 100644
index 00000000000..2d3e1d180c3
--- /dev/null
+++ b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
@@ -0,0 +1,733 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive_reader.h"
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+
+namespace doris::vectorized::table {
+
+class HiveReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create complex struct types for testing
+ void create_complex_struct_types(DataTypePtr& coordinates_struct_type,
+ DataTypePtr& address_struct_type,
+ DataTypePtr& phone_struct_type,
+ DataTypePtr& contact_struct_type,
+ DataTypePtr& hobby_element_struct_type,
+ DataTypePtr& hobbies_array_type,
+ DataTypePtr& profile_struct_type, DataTypePtr& name_type) {
+ // Create name column type (direct field)
+ name_type = make_nullable(std::make_shared<DataTypeString>());
+
+ // First create coordinates struct type
+ std::vector<DataTypePtr> coordinates_types = {
+ make_nullable(std::make_shared<DataTypeFloat64>()), // lat (field ID 10)
+ make_nullable(std::make_shared<DataTypeFloat64>()) // lng (field ID 11)
+ };
+ std::vector<std::string> coordinates_names = {"lat", "lng"};
+ coordinates_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(coordinates_types, coordinates_names));
+
+ // Create address struct type (with street, city, coordinates)
+ std::vector<DataTypePtr> address_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // street (field ID 7)
+ make_nullable(std::make_shared<DataTypeString>()), // city (field ID 8)
+ coordinates_struct_type // coordinates (field ID 9)
+ };
+ std::vector<std::string> address_names = {"street", "city", "coordinates"};
+ address_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(address_types, address_names));
+
+ // Create phone struct type
+ std::vector<DataTypePtr> phone_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // country_code (field ID 14)
+ make_nullable(std::make_shared<DataTypeString>()) // number (field ID 15)
+ };
+ std::vector<std::string> phone_names = {"country_code", "number"};
+ phone_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(phone_types, phone_names));
+
+ // Create contact struct type (with email, phone)
+ std::vector<DataTypePtr> contact_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // email (field ID 12)
+ phone_struct_type // phone (field ID 13)
+ };
+ std::vector<std::string> contact_names = {"email", "phone"};
+ contact_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(contact_types, contact_names));
+
+ // Create hobby element struct type for array elements
+ std::vector<DataTypePtr> hobby_element_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // name (field ID 17)
+ make_nullable(std::make_shared<DataTypeInt32>()) // level (field ID 18)
+ };
+ std::vector<std::string> hobby_element_names = {"name", "level"};
+ hobby_element_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(hobby_element_types, hobby_element_names));
+
+ // Create hobbies array type
+ hobbies_array_type =
+ make_nullable(std::make_shared<DataTypeArray>(hobby_element_struct_type));
+
+ // Create complete profile struct type (with address, contact, hobbies)
+ std::vector<DataTypePtr> profile_types = {
+ address_struct_type, // address (field ID 4)
+ contact_struct_type, // contact (field ID 5)
+ hobbies_array_type // hobbies (field ID 6)
+ };
+ std::vector<std::string> profile_names = {"address", "contact", "hobbies"};
+ profile_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(profile_types, profile_names));
+ }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [](TDescriptorTable& t_desc_table, TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::OLAP_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY nodes set contains_nulls; SCALAR nodes do not
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // Regular type
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+ // Set column_access_paths only for the profile field
+ if (table_column_names[i] == "profile") {
+ {
+ TColumnAccessPaths access_paths;
+ access_paths.__set_type(doris::TAccessPathType::NAME);
+ std::vector<TColumnNameAccessPath> paths;
+ // address.coordinates.lat
+ TColumnNameAccessPath path1;
+ path1.__set_path({"profile", "address", "coordinates", "lat"});
+ paths.push_back(path1);
+ // address.coordinates.lng
+ TColumnNameAccessPath path2;
+ path2.__set_path({"profile", "address", "coordinates", "lng"});
+ paths.push_back(path2);
+ // contact.email
+ TColumnNameAccessPath path3;
+ path3.__set_path({"profile", "contact", "email"});
+ paths.push_back(path3);
+ // hobbies[].element.level
+ TColumnNameAccessPath path4;
+ path4.__set_path({"profile", "hobbies", "*", "level"});
+ paths.push_back(path4);
+ access_paths.__set_name_access_paths(paths);
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+ }
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function to verify test results
+ void verify_test_results(Block& block, size_t read_rows) {
+ // Verify that we read some data
+ EXPECT_GT(read_rows, 0) << "Should read at least one row";
+ EXPECT_EQ(block.rows(), read_rows);
+
+ // Verify column count matches expected (2 columns: name, profile)
+ EXPECT_EQ(block.columns(), 2);
+
+ // Verify column names and types
+ auto columns_with_names = block.get_columns_with_type_and_name();
+ std::vector<std::string> expected_column_names = {"name", "profile"};
+ for (size_t i = 0; i < expected_column_names.size(); i++) {
+ EXPECT_EQ(columns_with_names[i].name, expected_column_names[i]);
+ }
+
+ // Verify column types
+ EXPECT_TRUE(columns_with_names[0].type->get_name().find("String") !=
+ std::string::npos); // name is STRING
+ EXPECT_TRUE(columns_with_names[1].type->get_name().find("Struct") !=
+ std::string::npos); // profile is STRUCT
+
+ // Print row count for each column and nested subcolumns
+ std::cout << "Block rows: " << block.rows() << std::endl;
+
+ // Helper function to recursively print column row counts
+ std::function<void(const ColumnPtr&, const DataTypePtr&, const std::string&, int)>
+ print_column_rows = [&](const ColumnPtr& col, const DataTypePtr& type,
+ const std::string& name, int depth) {
+ std::string indent(depth * 2, ' ');
+ std::cout << indent << name << " row count: " << col->size() << std::endl;
+
+ // Check if it's a nullable column
+ if (const auto* nullable_col = typeid_cast<const ColumnNullable*>(col.get())) {
+ auto nested_type =
+ assert_cast<const DataTypeNullable*>(type.get())->get_nested_type();
+
+ // Only add ".nested" suffix for non-leaf (complex) nullable columns
+ // Leaf columns like String, Int, etc. should not get the ".nested" suffix
+ bool is_complex_type =
+ (typeid_cast<const DataTypeStruct*>(nested_type.get()) !=
+ nullptr) ||
+ (typeid_cast<const DataTypeArray*>(nested_type.get()) != nullptr) ||
+ (typeid_cast<const DataTypeMap*>(nested_type.get()) != nullptr);
+
+ std::string nested_name = is_complex_type ? name + ".nested" : name;
+ print_column_rows(nullable_col->get_nested_column_ptr(), nested_type,
+ nested_name, depth + (is_complex_type ? 1 : 0));
+ }
+ // Check if it's a struct column
+ else if (const auto* struct_col = typeid_cast<const ColumnStruct*>(col.get())) {
+ auto struct_type = assert_cast<const DataTypeStruct*>(type.get());
+ for (size_t i = 0; i < struct_col->tuple_size(); ++i) {
+ std::string field_name = struct_type->get_element_name(i);
+ auto field_type = struct_type->get_element(i);
+ print_column_rows(struct_col->get_column_ptr(i), field_type,
+ name + "." + field_name, depth + 1);
+ }
+ }
+ // Check if it's an array column
+ else if (const auto* array_col = typeid_cast<const ColumnArray*>(col.get())) {
+ auto array_type = assert_cast<const DataTypeArray*>(type.get());
+ auto element_type = array_type->get_nested_type();
+ print_column_rows(array_col->get_data_ptr(), element_type, name + ".data",
+ depth + 1);
+ }
+ };
+
+ // Print row counts for all columns
+ for (size_t i = 0; i < block.columns(); ++i) {
+ const auto& column_with_name = block.get_by_position(i);
+ print_column_rows(column_with_name.column, column_with_name.type, column_with_name.name,
+ 0);
+ EXPECT_EQ(column_with_name.column->size(), block.rows())
+ << "Column " << column_with_name.name << " size mismatch";
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+};
+
+// Test reading real Hive Parquet file using HiveTableReader
+TEST_F(HiveReaderTest, ReadHiveParquetFile) {
+ // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
+ // Setup table descriptor for test columns with new schema:
+ /**
+ Schema:
+ message table {
+ required int64 id = 1;
+ required binary name (STRING) = 2;
+ required group profile = 3 {
+ optional group address = 4 {
+ optional binary street (STRING) = 7;
+ optional binary city (STRING) = 8;
+ optional group coordinates = 9 {
+ optional double lat = 10;
+ optional double lng = 11;
+ }
+ }
+ optional group contact = 5 {
+ optional binary email (STRING) = 12;
+ optional group phone = 13 {
+ optional binary country_code (STRING) = 14;
+ optional binary number (STRING) = 15;
+ }
+ }
+ optional group hobbies (LIST) = 6 {
+ repeated group list {
+ optional group element = 16 {
+ optional binary name (STRING) = 17;
+ optional int32 level = 18;
+ }
+ }
+ }
+ }
+ }
+ */
+
+ // Open the Hive Parquet test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ std::string test_file =
+ "./be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/"
+ "00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet";
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ GTEST_SKIP() << "Test file not found: " << test_file;
+ return;
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create ParquetReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader = ParquetReader::create_unique(&profile, scan_params, scan_range, 1024,
+ &ctz, nullptr, &runtime_state, cache.get());
+ ASSERT_NE(generic_reader, nullptr);
+
+ // Set file reader for the generic reader
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ // Create HiveParquetReader
+ auto hive_reader = std::make_unique<HiveParquetReader>(std::move(generic_reader), &profile,
+ &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ // Create complex struct types using helper function
+ DataTypePtr coordinates_struct_type, address_struct_type, phone_struct_type;
+ DataTypePtr contact_struct_type, hobby_element_struct_type, hobbies_array_type;
+ DataTypePtr profile_struct_type, name_type;
+ create_complex_struct_types(coordinates_struct_type, address_struct_type, phone_struct_type,
+ contact_struct_type, hobby_element_struct_type, hobbies_array_type,
+ profile_struct_type, name_type);
+
+ // Create tuple descriptor using helper function
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+ std::vector<std::string> table_column_names = {"name", "profile"};
+ std::vector<int> table_column_positions = {1, 2};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ TPrimitiveType::STRING, TPrimitiveType::STRUCT // profile 用 STRUCT 类型
+ };
+ const TupleDescriptor* tuple_descriptor =
+ create_tuple_descriptor(&desc_tbl, obj_pool, t_desc_table, t_table_desc,
+ table_column_names, table_column_positions, table_column_types);
+
+ VExprContextSPtrs conjuncts; // Empty conjuncts for this test
+ std::vector<std::string> table_col_names = {"name", "profile"};
+ const std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range = nullptr;
+ const RowDescriptor* row_descriptor = nullptr;
+ const std::unordered_map<std::string, int>* colname_to_slot_id = nullptr;
+ const VExprContextSPtrs* not_single_slot_filter_conjuncts = nullptr;
+ const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts = nullptr;
+
+ st = hive_reader->init_reader(table_col_names, colname_to_value_range, conjuncts,
+ tuple_descriptor, row_descriptor, colname_to_slot_id,
+ not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+ ASSERT_TRUE(hive_reader->set_fill_columns(partition_columns, missing_columns).ok());
+
+ // Create block for reading nested structure (not flattened)
+ Block block;
+ {
+ MutableColumnPtr name_column = name_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(name_column), name_type, "name"));
+ // Add profile column (nested struct)
+ MutableColumnPtr profile_column = profile_struct_type->create_column();
+ block.insert(
+ ColumnWithTypeAndName(std::move(profile_column), profile_struct_type, "profile"));
+ }
+
+ // Read data from the file
+ size_t read_rows = 0;
+ bool eof = false;
+ st = hive_reader->get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // Verify test results using helper function
+ verify_test_results(block, read_rows);
+}
+
+// Test reading real Hive Orc file using HiveTableReader
+TEST_F(HiveReaderTest, ReadHiveOrcFile) {
+ // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
+ // Setup table descriptor for test columns with new schema:
+ /**
+ Schema:
+ message table {
+ required int64 id = 1;
+ required binary name (STRING) = 2;
+ required group profile = 3 {
+ optional group address = 4 {
+ optional binary street (STRING) = 7;
+ optional binary city (STRING) = 8;
+ optional group coordinates = 9 {
+ optional double lat = 10;
+ optional double lng = 11;
+ }
+ }
+ optional group contact = 5 {
+ optional binary email (STRING) = 12;
+ optional group phone = 13 {
+ optional binary country_code (STRING) = 14;
+ optional binary number (STRING) = 15;
+ }
+ }
+ optional group hobbies (LIST) = 6 {
+ repeated group list {
+ optional group element = 16 {
+ optional binary name (STRING) = 17;
+ optional int32 level = 18;
+ }
+ }
+ }
+ }
+ }
+ */
+ // Open the Hive Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ std::string test_file =
+ "./be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/"
+ "00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc";
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ GTEST_SKIP() << "Test file not found: " << test_file;
+ return;
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader = OrcReader::create_unique(&profile, &runtime_state, scan_params,
+ scan_range, 1024, "CST", nullptr, cache.get());
+ ASSERT_NE(generic_reader, nullptr);
+
+ // Create HiveOrcReader
+ auto hive_reader =
+ std::make_unique<HiveOrcReader>(std::move(generic_reader), &profile, &runtime_state,
+ scan_params, scan_range, nullptr, nullptr, cache.get());
+
+ // Create complex struct types using helper function
+ DataTypePtr coordinates_struct_type, address_struct_type, phone_struct_type;
+ DataTypePtr contact_struct_type, hobby_element_struct_type, hobbies_array_type;
+ DataTypePtr profile_struct_type, name_type;
+ create_complex_struct_types(coordinates_struct_type, address_struct_type, phone_struct_type,
+ contact_struct_type, hobby_element_struct_type, hobbies_array_type,
+ profile_struct_type, name_type);
+
+ // Create tuple descriptor using helper function
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+ std::vector<std::string> table_column_names = {"name", "profile"};
+ std::vector<int> table_column_positions = {1, 2};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ TPrimitiveType::STRING, TPrimitiveType::STRUCT // profile 用 STRUCT 类型
+ };
+ const TupleDescriptor* tuple_descriptor =
+ create_tuple_descriptor(&desc_tbl, obj_pool, t_desc_table, t_table_desc,
+ table_column_names, table_column_positions, table_column_types);
+
+ VExprContextSPtrs conjuncts; // Empty conjuncts for this test
+ std::vector<std::string> table_col_names = {"name", "profile"};
+ const std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range = nullptr;
+ const RowDescriptor* row_descriptor = nullptr;
+ const VExprContextSPtrs* not_single_slot_filter_conjuncts = nullptr;
+ const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts = nullptr;
+
+ st = hive_reader->init_reader(table_col_names, colname_to_value_range, conjuncts,
+ tuple_descriptor, row_descriptor,
+ not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+ ASSERT_TRUE(hive_reader->set_fill_columns(partition_columns, missing_columns).ok());
+
+ // Create block for reading nested structure (not flattened)
+ Block block;
+
+ {
+ MutableColumnPtr name_column = name_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(name_column), name_type, "name"));
+ // Add profile column (nested struct)
+ MutableColumnPtr profile_column = profile_struct_type->create_column();
+ block.insert(
+ ColumnWithTypeAndName(std::move(profile_column), profile_struct_type, "profile"));
+ }
+
+ // Read data from the file
+ size_t read_rows = 0;
+ bool eof = false;
+ st = hive_reader->get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // Verify test results using helper function
+ verify_test_results(block, read_rows);
+}
+
+} // namespace doris::vectorized::table
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
new file mode 100644
index 00000000000..e5d1aabcaed
--- /dev/null
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
@@ -0,0 +1,1181 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+#include "vec/exec/format/table/iceberg_reader.h"
+
+namespace doris::vectorized {
+
+// Define the column access path configuration structure
+struct ColumnAccessPathConfig {
+ std::string column_name;
+ std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
+ std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
+ bool use_name_paths = true; // true=use field names, false=use indices
+};
+
+// Iceberg 列ID分配 (基于 Protobuf Schema 定义)
+// 1: id (int64)
+// 2: name (string)
+// 3: profile (struct)
+// 9: address (struct)
+// 12: street (string)
+// 13: city (string)
+// 14: coordinates (struct)
+// 15: lat (double)
+// 16: lng (double)
+// 10: contact (struct)
+// 17: email (string)
+// 18: phone (struct)
+// 19: country_code (string)
+// 20: number (string)
+// 11: hobbies (list)
+// 21: element (struct)
+// 22: name (string)
+// 23: level (int32)
+// 4: tags (list)
+// 24: element (string)
+// 5: friends (list)
+// 25: element (struct)
+// 26: user_id (int64)
+// 27: nickname (string)
+// 28: friendship_level (int32)
+// 6: recent_activity (list)
+// 29: element (struct)
+// 30: action (string)
+// 31: details (list)
+// 32: element (struct)
+// 33: key (string)
+// 34: value (string)
+// 7: attributes (map)
+// 35: key (string)
+// 36: value (string)
+// 8: complex_attributes (map)
+// 37: key (string)
+// 38: value (struct)
+// 39: metadata (struct)
+// 43: version (string)
+// 44: created_time (timestamp)
+// 45: last_updated (timestamp)
+// 46: quality_score (double)
+// 40: historical_scores (list)
+// 47: element (struct)
+// 48: period (string)
+// 49: score (double)
+// 50: components (list)
+// 52: element (struct)
+// 53: component_name (string)
+// 54: weight (float)
+// 55: sub_scores (map)
+// 56: key (string)
+// 57: value (double)
+// 51: trends (struct)
+// 58: direction (string)
+// 59: magnitude (double)
+// 60: confidence_interval (struct)
+// 61: lower (double)
+// 62: upper (double)
+// 41: hierarchical_data (map)
+// 63: key (string)
+// 64: value (struct)
+// 65: category (string)
+// 66: sub_items (list)
+// 68: element (struct)
+// 69: item_id (int64)
+// 70: properties (map)
+// 72: key (string)
+// 73: value (string)
+// 71: metrics (struct)
+// 74: value (double)
+// 75: unit (string)
+// 67: summary (struct)
+// 76: total_count (int32)
+// 77: average_value (double)
+// 42: validation_rules (struct)
+// 78: required (boolean)
+// 79: constraints (list)
+// 81: element (struct)
+// 82: rule_type (string)
+// 83: parameters (map)
+// 85: key (string)
+// 86: value (string)
+// 84: error_message (string)
+// 80: complex_constraint (struct)
+// 87: logic_operator (string)
+// 88: operands (list)
+// 89: element (struct)
+// 90: field_path (string)
+// 91: operator (string)
+// 92: comparison_value (string)
+
+class IcebergReaderCreateColumnIdsTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to map column name to Iceberg field ID
+ int get_iceberg_field_id(const std::string& column_name) {
+ // Mapping based on the Iceberg schema definition in comments
+ static const std::unordered_map<std::string, int> column_to_field_id = {
+ {"id", 1}, {"name", 2},
+ {"profile", 3}, {"tags", 4},
+ {"friends", 5}, {"recent_activity", 6},
+ {"attributes", 7}, {"complex_attributes", 8}};
+
+ auto it = column_to_field_id.find(column_name);
+ if (it != column_to_field_id.end()) {
+ return it->second;
+ }
+ return -1; // Invalid field ID
+ }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types,
+ const std::vector<ColumnAccessPathConfig>& access_configs = {}) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [this, &access_configs](
+ TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::HIVE_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "id") {
+ // id: bigint
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::BIGINT);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "name") {
+ // name: string
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::STRING);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "tags") {
+ // tags: array<string>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_node;
+ element_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType element_scalar;
+ element_scalar.__set_type(TPrimitiveType::STRING);
+ element_node.__set_scalar_type(element_scalar);
+ type.types.push_back(element_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "friends") {
+ // friends: array<struct<user_id:bigint, nickname:string, friendship_level:int>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField user_id_field;
+ user_id_field.__set_name("user_id");
+ user_id_field.__set_contains_null(true);
+ element_fields.push_back(user_id_field);
+ TStructField nickname_field;
+ nickname_field.__set_name("nickname");
+ nickname_field.__set_contains_null(true);
+ element_fields.push_back(nickname_field);
+ TStructField friendship_level_field;
+ friendship_level_field.__set_name("friendship_level");
+ friendship_level_field.__set_contains_null(true);
+ element_fields.push_back(friendship_level_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // user_id: bigint
+ TTypeNode user_id_node;
+ user_id_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType user_id_scalar;
+ user_id_scalar.__set_type(TPrimitiveType::BIGINT);
+ user_id_node.__set_scalar_type(user_id_scalar);
+ type.types.push_back(user_id_node);
+ // nickname: string
+ TTypeNode nickname_node;
+ nickname_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType nickname_scalar;
+ nickname_scalar.__set_type(TPrimitiveType::STRING);
+ nickname_node.__set_scalar_type(nickname_scalar);
+ type.types.push_back(nickname_node);
+ // friendship_level: int
+ TTypeNode friendship_level_node;
+ friendship_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType friendship_level_scalar;
+ friendship_level_scalar.__set_type(TPrimitiveType::INT);
+ friendship_level_node.__set_scalar_type(friendship_level_scalar);
+ type.types.push_back(friendship_level_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "recent_activity") {
+ // recent_activity: array<struct<action:string, details:array<struct<key:string, value:string>>>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField action_field;
+ action_field.__set_name("action");
+ action_field.__set_contains_null(true);
+ element_fields.push_back(action_field);
+ TStructField details_field;
+ details_field.__set_name("details");
+ details_field.__set_contains_null(true);
+ element_fields.push_back(details_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // action: string
+ TTypeNode action_node;
+ action_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType action_scalar;
+ action_scalar.__set_type(TPrimitiveType::STRING);
+ action_node.__set_scalar_type(action_scalar);
+ type.types.push_back(action_node);
+ // details: array<struct<key:string, value:string>>
+ TTypeNode details_array_node;
+ details_array_node.__set_type(TTypeNodeType::ARRAY);
+ details_array_node.__set_contains_nulls({true});
+ type.types.push_back(details_array_node);
+ TTypeNode details_element_struct_node;
+ details_element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> details_element_fields;
+ TStructField key_field;
+ key_field.__set_name("key");
+ key_field.__set_contains_null(true);
+ details_element_fields.push_back(key_field);
+ TStructField value_field;
+ value_field.__set_name("value");
+ value_field.__set_contains_null(true);
+ details_element_fields.push_back(value_field);
+ details_element_struct_node.__set_struct_fields(details_element_fields);
+ type.types.push_back(details_element_struct_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "attributes") {
+ // attributes: map<string, string>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "complex_attributes") {
+ // complex_attributes: map<string, struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>
+ TTypeNode value_struct_node;
+ value_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> value_fields;
+ TStructField metadata_field;
+ metadata_field.__set_name("metadata");
+ metadata_field.__set_contains_null(true);
+ value_fields.push_back(metadata_field);
+ value_struct_node.__set_struct_fields(value_fields);
+ type.types.push_back(value_struct_node);
+ // metadata: struct<version:string, created_time:timestamp, last_updated:timestamp>
+ TTypeNode metadata_struct_node;
+ metadata_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> metadata_fields;
+ TStructField version_field;
+ version_field.__set_name("version");
+ version_field.__set_contains_null(true);
+ metadata_fields.push_back(version_field);
+ TStructField created_time_field;
+ created_time_field.__set_name("created_time");
+ created_time_field.__set_contains_null(true);
+ metadata_fields.push_back(created_time_field);
+ TStructField last_updated_field;
+ last_updated_field.__set_name("last_updated");
+ last_updated_field.__set_contains_null(true);
+ metadata_fields.push_back(last_updated_field);
+ metadata_struct_node.__set_struct_fields(metadata_fields);
+ type.types.push_back(metadata_struct_node);
+ // version: string
+ TTypeNode version_node;
+ version_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType version_scalar;
+ version_scalar.__set_type(TPrimitiveType::STRING);
+ version_node.__set_scalar_type(version_scalar);
+ type.types.push_back(version_node);
+ // created_time: timestamp
+ TTypeNode created_time_node;
+ created_time_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType created_time_scalar;
+ created_time_scalar.__set_type(TPrimitiveType::DATETIME);
+ created_time_node.__set_scalar_type(created_time_scalar);
+ type.types.push_back(created_time_node);
+ // last_updated: timestamp
+ TTypeNode last_updated_node;
+ last_updated_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType last_updated_scalar;
+ last_updated_scalar.__set_type(TPrimitiveType::DATETIME);
+ last_updated_node.__set_scalar_type(last_updated_scalar);
+ type.types.push_back(last_updated_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY 节点设置 contains_nulls，SCALAR 节点不设置
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // 普通类型
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+
+ // Set Iceberg field ID (col_unique_id)
+ int iceberg_field_id = get_iceberg_field_id(table_column_names[i]);
+ if (iceberg_field_id > 0) {
+ tslot_desc.__set_col_unique_id(iceberg_field_id);
+ }
+
+ // Use configuration to set column_access_paths
+ for (const auto& config : access_configs) {
+ set_column_access_paths(tslot_desc, config);
+ }
+
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function: set column access paths on a slot descriptor
+ void set_column_access_paths(TSlotDescriptor& tslot_desc,
+ const ColumnAccessPathConfig& config) {
+ if (config.column_name != tslot_desc.colName) {
+ return; // Not the target column, skip
+ }
+
+ // Set all_column_access_paths
+ if (!config.all_column_paths.empty()) {
+ TColumnAccessPaths access_paths;
+ access_paths.__set_type(doris::TAccessPathType::NAME);
+ std::vector<TColumnNameAccessPath> paths;
+
+ for (const auto& path_vector : config.all_column_paths) {
+ TColumnNameAccessPath path;
+ path.__set_path(path_vector);
+ paths.push_back(path);
+ }
+
+ access_paths.__set_name_access_paths(paths);
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+
+ // Set predicate_column_access_paths
+ if (!config.predicate_paths.empty()) {
+ TColumnAccessPaths access_paths;
+ access_paths.__set_type(doris::TAccessPathType::NAME);
+ std::vector<TColumnNameAccessPath> paths;
+
+ for (const auto& path_vector : config.predicate_paths) {
+ TColumnNameAccessPath path;
+ path.__set_path(path_vector);
+ paths.push_back(path);
+ }
+
+ access_paths.__set_name_access_paths(paths);
+ tslot_desc.__set_predicate_access_paths(access_paths);
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+
+ // Helper function: create and setup ParquetReader
+ std::tuple<std::unique_ptr<IcebergParquetReader>, const FieldDescriptor*> create_parquet_reader(
+ const std::string& test_file) {
+ // Open the Iceberg Parquet test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create ParquetReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ ParquetReader::create_unique(&profile, scan_params, scan_range, 1024, &ctz, nullptr,
+ &runtime_state, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ // Set file reader for the generic reader
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ const FieldDescriptor* field_desc = nullptr;
+ st = parquet_reader->get_file_metadata_schema(&field_desc);
+ if (!st.ok() || !field_desc) {
+ return {nullptr, nullptr};
+ }
+
+ // Create IcebergParquetReader
+ auto iceberg_reader = std::make_unique<IcebergParquetReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(iceberg_reader), field_desc};
+ }
+
+ // Helper function: create and setup OrcReader
+ std::tuple<std::unique_ptr<IcebergOrcReader>, const orc::Type*> create_orc_reader(
+ const std::string& test_file) {
+ // Open the Iceberg Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ OrcReader::create_unique(&profile, &runtime_state, scan_params, scan_range, 1024,
+ "CST", nullptr, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto orc_reader = static_cast<OrcReader*>(generic_reader.get());
+ // Get FieldDescriptor from Orc file
+ const orc::Type* orc_type_ptr = nullptr;
+ st = orc_reader->get_file_type(&orc_type_ptr);
+ if (!st.ok() || !orc_type_ptr) {
+ return {nullptr, nullptr};
+ }
+
+ // Create IcebergOrcReader
+ auto iceberg_reader = std::make_unique<IcebergOrcReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(iceberg_reader), orc_type_ptr};
+ }
+
+ // Helper function: run Parquet test with different column ID extraction methods
+ void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/"
+ "00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet";
+
+ auto [iceberg_reader, field_desc] = create_parquet_reader(test_file);
+ if (!iceberg_reader || !field_desc) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ // actual_result = IcebergParquetReader::_create_column_ids_by_top_level_col_index(
+ // field_desc, tuple_descriptor);
+ } else {
+ actual_result = IcebergParquetReader::_create_column_ids(field_desc, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+
+ // Helper function: run Orc test with different column ID extraction methods
+ void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/"
+ "00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc";
+
+ auto [iceberg_reader, orc_type] = create_orc_reader(test_file);
+ if (!iceberg_reader || !orc_type) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ // actual_result = IcebergOrcReader::_create_column_ids_by_top_level_col_index(
+ // orc_type, tuple_descriptor);
+ } else {
+ actual_result = IcebergOrcReader::_create_column_ids(orc_type, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+};
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_1) {
+ // Properties:
+
+ // Configure access paths for the profile column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3", "9", "14", "15"},
+ {"3", "9", "14", "16"},
+ {"3", "10", "17"},
+ {"3", "11", "*", "23"}};
+ access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_2) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3"}};
+ access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_3) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3", "10"}, {"3", "9"}};
+ access_config.predicate_paths = {{"3", "9", "14"}, {"3", "10", "17"}};
+
+ // access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
+ // access_config.predicate_paths = {{"profile", "address", "coordinates"},
+ // {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_4) {
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {};
+ access_config.predicate_paths = {};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // 19: tags (array)
+ // 20: element (string)
+ // 21: friends (array)
+ // 22: element (struct)
+ // 23: user_id (bigint)
+ // 24: nickname (string)
+ // 25: friendship_level (int)
+ // 26: recent_activity (array)
+ // 27: element (struct)
+ // 28: action (string)
+ // 29: details (array)
+ // 30: element (struct)
+ // 31: key (string)
+ // 32: value (string)
+
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // 配置friends列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "friends";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"5", "*", "27"}, {"5", "*", "28"}};
+ access_config.predicate_paths = {
+ {"5", "*", "27"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // 配置recent_activity列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "recent_activity";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"6", "*", "30"}, {"6", "*", "31", "*", "34"}};
+ access_config.predicate_paths = {
+ {"6", "*", "30"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // 33: attributes (map)
+ // 34: key (string)
+ // 35: value (string)
+ // 36: complex_attributes (map)
+ // 37: key (string)
+ // 38: value (struct)
+ // 39: metadata (struct)
+ // 40: version (string)
+ // 41: created_time (timestamp)
+ // 42: last_updated (timestamp)
+ // 43: quality_score (double)
+ // 44: historical_scores (array)
+ // 45: element (struct)
+ // 46: period (string)
+ // 47: score (double)
+ // 48: components (array)
+ // 49: element (struct)
+ // 50: component_name (string)
+ // 51: weight (float)
+ // 52: sub_scores (map)
+ // 53: key (string)
+ // 54: value (double)
+ // 55: trends (struct)
+ // 56: direction (string)
+ // 57: magnitude (double)
+ // 58: confidence_interval (struct)
+ // 59: lower (double)
+ // 60: upper (double)
+ // 61: hierarchical_data (map)
+ // 62: key (string)
+ // 63: value (struct)
+ // 64: category (string)
+ // 65: sub_items (array)
+ // 66: element (struct)
+ // 67: item_id (bigint)
+ // 68: properties (map)
+ // 69: key (string)
+ // 70: value (string)
+ // 71: metrics (struct)
+ // 72: value (double)
+ // 73: unit (string)
+ // 74: summary (struct)
+ // 75: total_count (int)
+ // 76: average_value (double)
+ // 77: validation_rules (struct)
+ // 78: required (boolean)
+ // 79: constraints (array)
+ // 80: element (struct)
+ // 81: rule_type (string)
+ // 82: parameters (map)
+ // 83: key (string)
+ // 84: value (string)
+ // 85: error_message (string)
+ // 86: complex_constraint (struct)
+ // 87: logic_operator (string)
+ // 88: operands (array)
+ // 89: element (struct)
+ // 90: field_path (string)
+ // 91: operator (string)
+ // 92: comparison_value (string)
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // 配置complex_attributes列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "complex_attributes";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"8", "*", "39", "43"},
+ {"8", "*", "40", "*", "50", "*", "55"},
+ {"8", "*", "41", "VALUES"},
+ {"8", "VALUES", "42", "79", "*", "83", "KEYS"}};
+ access_config.predicate_paths = {{"8", "*", "39", "43"}};
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // parquet values should access keys
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
+ 54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
+ 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ }
+
+ {
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
+ 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
+ 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
new file mode 100644
index 00000000000..8e4c37fd3b6
--- /dev/null
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
@@ -0,0 +1,729 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg_reader.h"
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+
+namespace doris::vectorized {
+
+class IcebergReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create complex struct types for testing
+ void create_complex_struct_types(DataTypePtr& coordinates_struct_type,
+ DataTypePtr& address_struct_type,
+ DataTypePtr& phone_struct_type,
+ DataTypePtr& contact_struct_type,
+ DataTypePtr& hobby_element_struct_type,
+ DataTypePtr& hobbies_array_type,
+ DataTypePtr& profile_struct_type, DataTypePtr& name_type) {
+ // Create name column type (direct field)
+ name_type = make_nullable(std::make_shared<DataTypeString>());
+
+ // First create coordinates struct type
+ std::vector<DataTypePtr> coordinates_types = {
+ make_nullable(std::make_shared<DataTypeFloat64>()), // lat (field ID 10)
+ make_nullable(std::make_shared<DataTypeFloat64>()) // lng (field ID 11)
+ };
+ std::vector<std::string> coordinates_names = {"lat", "lng"};
+ coordinates_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(coordinates_types, coordinates_names));
+
+ // Create address struct type (with street, city, coordinates)
+ std::vector<DataTypePtr> address_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // street (field ID 7)
+ make_nullable(std::make_shared<DataTypeString>()), // city (field ID 8)
+ coordinates_struct_type // coordinates (field ID 9)
+ };
+ std::vector<std::string> address_names = {"street", "city", "coordinates"};
+ address_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(address_types, address_names));
+
+ // Create phone struct type
+ std::vector<DataTypePtr> phone_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // country_code (field ID 14)
+ make_nullable(std::make_shared<DataTypeString>()) // number (field ID 15)
+ };
+ std::vector<std::string> phone_names = {"country_code", "number"};
+ phone_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(phone_types, phone_names));
+
+ // Create contact struct type (with email, phone)
+ std::vector<DataTypePtr> contact_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // email (field ID 12)
+ phone_struct_type // phone (field ID 13)
+ };
+ std::vector<std::string> contact_names = {"email", "phone"};
+ contact_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(contact_types, contact_names));
+
+ // Create hobby element struct type for array elements
+ std::vector<DataTypePtr> hobby_element_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // name (field ID 17)
+ make_nullable(std::make_shared<DataTypeInt32>()) // level (field ID 18)
+ };
+ std::vector<std::string> hobby_element_names = {"name", "level"};
+ hobby_element_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(hobby_element_types, hobby_element_names));
+
+ // Create hobbies array type
+ hobbies_array_type =
+ make_nullable(std::make_shared<DataTypeArray>(hobby_element_struct_type));
+
+ // Create complete profile struct type (with address, contact, hobbies)
+ std::vector<DataTypePtr> profile_types = {
+ address_struct_type, // address (field ID 4)
+ contact_struct_type, // contact (field ID 5)
+ hobbies_array_type // hobbies (field ID 6)
+ };
+ std::vector<std::string> profile_names = {"address", "contact", "hobbies"};
+ profile_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(profile_types, profile_names));
+ }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(DescriptorTbl** desc_tbl, ObjectPool& obj_pool,
+ TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc) {
+ std::vector<std::string> table_column_names = {"name", "profile"};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ TPrimitiveType::STRING, TPrimitiveType::STRUCT // profile uses STRUCT type
+ };
+
+ // Create table descriptor with complex schema
+ auto create_table_desc = [](TDescriptorTable& t_desc_table, TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::OLAP_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ // iceberg_id must correspond to table_column_names order
+ std::vector<int32_t> iceberg_ids = {2, 3}; // name:2, profile:3
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_col_unique_id(iceberg_ids[i]);
+ TTypeDesc type;
+ if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY nodes set contains_nulls; SCALAR nodes do not
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // Regular type
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_columnPos(0);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+ // Set column_access_paths only for the profile field
+ if (table_column_names[i] == "profile") {
+ TColumnAccessPaths access_paths;
+ access_paths.__set_type(doris::TAccessPathType::NAME);
+ std::vector<TColumnNameAccessPath> paths;
+ // address.coordinates.lat
+ TColumnNameAccessPath path1;
+ path1.__set_path({"3", "4", "9", "10"});
+ paths.push_back(path1);
+ // address.coordinates.lng
+ TColumnNameAccessPath path2;
+ path2.__set_path({"3", "4", "9", "11"});
+ paths.push_back(path2);
+ // contact.email
+ TColumnNameAccessPath path3;
+ path3.__set_path({"3", "5", "12"});
+ paths.push_back(path3);
+ // hobbies[].element.level
+ TColumnNameAccessPath path4;
+ path4.__set_path({"3", "6", "16", "18"});
+ paths.push_back(path4);
+ access_paths.__set_name_access_paths(paths);
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, table_column_names, table_column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function to verify test results
+ void verify_test_results(Block& block, size_t read_rows) {
+ // Verify that we read some data
+ EXPECT_GT(read_rows, 0) << "Should read at least one row";
+ EXPECT_EQ(block.rows(), read_rows);
+
+ // Verify column count matches expected (2 columns: name, profile)
+ EXPECT_EQ(block.columns(), 2);
+
+ // Verify column names and types
+ auto columns_with_names = block.get_columns_with_type_and_name();
+ std::vector<std::string> expected_column_names = {"name", "profile"};
+ for (size_t i = 0; i < expected_column_names.size(); i++) {
+ EXPECT_EQ(columns_with_names[i].name, expected_column_names[i]);
+ }
+
+ // Verify column types
+ EXPECT_TRUE(columns_with_names[0].type->get_name().find("String") !=
+ std::string::npos); // name is STRING
+ EXPECT_TRUE(columns_with_names[1].type->get_name().find("Struct") !=
+ std::string::npos); // profile is STRUCT
+
+ // Print row count for each column and nested subcolumns
+ std::cout << "Block rows: " << block.rows() << std::endl;
+
+ // Helper function to recursively print column row counts
+ std::function<void(const ColumnPtr&, const DataTypePtr&, const std::string&, int)>
+ print_column_rows = [&](const ColumnPtr& col, const DataTypePtr& type,
+ const std::string& name, int depth) {
+ std::string indent(depth * 2, ' ');
+ std::cout << indent << name << " row count: " << col->size() << std::endl;
+
+ // Check if it's a nullable column
+ if (const auto* nullable_col = typeid_cast<const ColumnNullable*>(col.get())) {
+ auto nested_type =
+ assert_cast<const DataTypeNullable*>(type.get())->get_nested_type();
+
+ // Only add ".nested" suffix for non-leaf (complex) nullable columns
+ // Leaf columns like String, Int, etc. should not get the ".nested" suffix
+ bool is_complex_type =
+ (typeid_cast<const DataTypeStruct*>(nested_type.get()) !=
+ nullptr) ||
+ (typeid_cast<const DataTypeArray*>(nested_type.get()) != nullptr) ||
+ (typeid_cast<const DataTypeMap*>(nested_type.get()) != nullptr);
+
+ std::string nested_name = is_complex_type ? name + ".nested" : name;
+ print_column_rows(nullable_col->get_nested_column_ptr(), nested_type,
+ nested_name, depth + (is_complex_type ? 1 : 0));
+ }
+ // Check if it's a struct column
+ else if (const auto* struct_col = typeid_cast<const ColumnStruct*>(col.get())) {
+ auto struct_type = assert_cast<const DataTypeStruct*>(type.get());
+ for (size_t i = 0; i < struct_col->tuple_size(); ++i) {
+ std::string field_name = struct_type->get_element_name(i);
+ auto field_type = struct_type->get_element(i);
+ print_column_rows(struct_col->get_column_ptr(i), field_type,
+ name + "." + field_name, depth + 1);
+ }
+ }
+ // Check if it's an array column
+ else if (const auto* array_col = typeid_cast<const ColumnArray*>(col.get())) {
+ auto array_type = assert_cast<const DataTypeArray*>(type.get());
+ auto element_type = array_type->get_nested_type();
+ print_column_rows(array_col->get_data_ptr(), element_type, name + ".data",
+ depth + 1);
+ }
+ };
+
+ // Print row counts for all columns
+ for (size_t i = 0; i < block.columns(); ++i) {
+ const auto& column_with_name = block.get_by_position(i);
+ print_column_rows(column_with_name.column, column_with_name.type, column_with_name.name,
+ 0);
+ EXPECT_EQ(column_with_name.column->size(), block.rows())
+ << "Column " << column_with_name.name << " size mismatch";
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+};
+
+// Test reading real Iceberg Parquet file using IcebergTableReader
+TEST_F(IcebergReaderTest, ReadIcebergParquetFile) {
+ // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
+ // Setup table descriptor for test columns with new schema:
+ /**
+ Properties:
+ iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type":{"type" [...]
+ Schema:
+ message table {
+ required int64 id = 1;
+ required binary name (STRING) = 2;
+ required group profile = 3 {
+ optional group address = 4 {
+ optional binary street (STRING) = 7;
+ optional binary city (STRING) = 8;
+ optional group coordinates = 9 {
+ optional double lat = 10;
+ optional double lng = 11;
+ }
+ }
+ optional group contact = 5 {
+ optional binary email (STRING) = 12;
+ optional group phone = 13 {
+ optional binary country_code (STRING) = 14;
+ optional binary number (STRING) = 15;
+ }
+ }
+ optional group hobbies (LIST) = 6 {
+ repeated group list {
+ optional group element = 16 {
+ optional binary name (STRING) = 17;
+ optional int32 level = 18;
+ }
+ }
+ }
+ }
+ }
+ */
+
+ // Open the Iceberg Parquet test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ std::string test_file =
+ "./be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/"
+ "00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet";
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ GTEST_SKIP() << "Test file not found: " << test_file;
+ return;
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create ParquetReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader = ParquetReader::create_unique(&profile, scan_params, scan_range, 1024,
+ &ctz, nullptr, &runtime_state, cache.get());
+ ASSERT_NE(generic_reader, nullptr);
+
+ // Set file reader for the generic reader
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ // Create IcebergParquetReader
+ auto iceberg_reader = std::make_unique<IcebergParquetReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range, nullptr,
+ nullptr, cache.get());
+
+ // Create complex struct types using helper function
+ DataTypePtr coordinates_struct_type, address_struct_type, phone_struct_type;
+ DataTypePtr contact_struct_type, hobby_element_struct_type, hobbies_array_type;
+ DataTypePtr profile_struct_type, name_type;
+ create_complex_struct_types(coordinates_struct_type, address_struct_type, phone_struct_type,
+ contact_struct_type, hobby_element_struct_type, hobbies_array_type,
+ profile_struct_type, name_type);
+
+ // Create tuple descriptor using helper function
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+ const TupleDescriptor* tuple_descriptor =
+ create_tuple_descriptor(&desc_tbl, obj_pool, t_desc_table, t_table_desc);
+
+ VExprContextSPtrs conjuncts; // Empty conjuncts for this test
+ std::vector<std::string> table_col_names = {"name", "profile"};
+ const std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range = nullptr;
+ const RowDescriptor* row_descriptor = nullptr;
+ const std::unordered_map<std::string, int>* colname_to_slot_id = nullptr;
+ const VExprContextSPtrs* not_single_slot_filter_conjuncts = nullptr;
+ const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts = nullptr;
+
+ st = iceberg_reader->init_reader(table_col_names, colname_to_value_range, conjuncts,
+ tuple_descriptor, row_descriptor, colname_to_slot_id,
+ not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+ ASSERT_TRUE(iceberg_reader->set_fill_columns(partition_columns, missing_columns).ok());
+
+ // Create block for reading nested structure (not flattened)
+ Block block;
+ {
+ MutableColumnPtr name_column = name_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(name_column), name_type, "name"));
+ // Add profile column (nested struct)
+ MutableColumnPtr profile_column = profile_struct_type->create_column();
+ block.insert(
+ ColumnWithTypeAndName(std::move(profile_column), profile_struct_type, "profile"));
+ }
+
+ // Read data from the file
+ size_t read_rows = 0;
+ bool eof = false;
+ st = iceberg_reader->get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // Verify test results using helper function
+ verify_test_results(block, read_rows);
+}
+
... 265 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org