You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2025/10/31 04:46:45 UTC
(doris) 02/13: [Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ae92dbce84100235b21538c8acf89564687dfe8d
Author: kakachen <ch...@selectdb.com>
AuthorDate: Thu Oct 23 09:54:32 2025 +0800
[Feature](catalog) Prune nested columns for hive/iceberg parquet and orc reader.
---
.gitmodules | 2 +-
be/src/runtime/descriptors.h | 6 +
be/src/vec/exec/format/orc/vorc_reader.cpp | 436 ++++++--
be/src/vec/exec/format/orc/vorc_reader.h | 24 +-
be/src/vec/exec/format/parquet/schema_desc.cpp | 87 +-
be/src/vec/exec/format/parquet/schema_desc.h | 30 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 264 ++++-
.../exec/format/parquet/vparquet_column_reader.h | 105 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 40 +-
.../exec/format/parquet/vparquet_group_reader.h | 8 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 21 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 5 +-
.../table/hive/hive_orc_nested_column_utils.cpp | 148 +++
.../table/hive/hive_orc_nested_column_utils.h | 43 +
.../hive/hive_parquet_nested_column_utils.cpp | 172 +++
.../table/hive/hive_parquet_nested_column_utils.h | 41 +
be/src/vec/exec/format/table/hive_reader.cpp | 453 +++++++-
be/src/vec/exec/format/table/hive_reader.h | 14 +
.../iceberg/iceberg_orc_nested_column_utils.cpp | 149 +++
.../iceberg/iceberg_orc_nested_column_utils.h | 46 +
.../iceberg_parquet_nested_column_utils.cpp | 173 +++
.../iceberg/iceberg_parquet_nested_column_utils.h | 42 +
be/src/vec/exec/format/table/iceberg_reader.cpp | 253 ++++-
be/src/vec/exec/format/table/iceberg_reader.h | 7 +
.../vec/exec/format/table/table_format_reader.cpp | 21 +-
be/src/vec/exec/format/table/table_format_reader.h | 36 +
be/src/vec/exec/scan/file_scanner.cpp | 6 +-
be/src/vec/exec/scan/file_scanner.h | 2 +-
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2277 bytes
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2334 bytes
...2c4c8-e0bd-4db6-b550-6323977446a4.metadata.json | 145 +++
...66011-3129-4bfb-9b4b-c1510c566d73.metadata.json | 176 +++
.../6a876742-85c2-4f84-ae24-292482fa9288-m0.avro | Bin 0 -> 8175 bytes
...212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro | Bin 0 -> 4264 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4068 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4020 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4043 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4082 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4067 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4047 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4077 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4147 bytes
...38ed9-31af-46c8-9274-506c7f2709c9.metadata.json | 145 +++
...4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json | 176 +++
.../51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro | Bin 0 -> 8783 bytes
...552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro | Bin 0 -> 4264 bytes
...a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc | Bin 0 -> 7181 bytes
...20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json | 548 +++++++++
...ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json | 579 ++++++++++
.../d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro | 1 +
...228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro | Bin 0 -> 4261 bytes
...35f-d581-40e4-8d56-a929aded99e4-0-00001.parquet | Bin 0 -> 21586 bytes
...ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json | 548 +++++++++
...56265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json | 579 ++++++++++
.../bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro | 5 +
...002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro | Bin 0 -> 4262 bytes
.../test_data/nested_user_profiles_orc/_SUCCESS | 0
...f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc | Bin 0 -> 8396 bytes
.../nested_user_profiles_parquet/_SUCCESS | 0
...1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet | Bin 0 -> 21157 bytes
.../exec/format/parquet/parquet_thrift_test.cpp | 12 +-
.../hive/hive_reader_create_column_ids_test.cpp | 1179 ++++++++++++++++++++
.../exec/format/table/hive/hive_reader_test.cpp | 743 ++++++++++++
.../iceberg_reader_create_column_ids_test.cpp | 1177 +++++++++++++++++++
.../format/table/iceberg/iceberg_reader_test.cpp | 739 ++++++++++++
.../org/apache/doris/datasource/FileScanNode.java | 1 -
.../trees/plans/logical/LogicalFileScan.java | 19 +-
.../trees/plans/logical/LogicalHudiScan.java | 2 +-
68 files changed, 9229 insertions(+), 179 deletions(-)
diff --git a/.gitmodules b/.gitmodules
index 54c1a8a3636..63e4b14f70d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -20,7 +20,7 @@
[submodule "contrib/apache-orc"]
path = contrib/apache-orc
url = https://github.com/apache/doris-thirdparty.git
- branch = orc
+ branch = cq_nested_column_prune_external_table
[submodule "doris-faiss"]
path = contrib/faiss
url = https://github.com/apache/doris-thirdparty.git
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index c66ccd80b1d..b34e54622b5 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -100,6 +100,10 @@ public:
return virtual_column_expr;
}
+ void set_is_predicate(bool is_predicate) { _is_predicate = is_predicate; }
+
+ bool is_predicate() const { return _is_predicate; }
+
private:
friend class DescriptorTbl;
friend class TupleDescriptor;
@@ -141,6 +145,8 @@ private:
std::shared_ptr<doris::TExpr> virtual_column_expr = nullptr;
+ bool _is_predicate = false;
+
SlotDescriptor(const TSlotDescriptor& tdesc);
SlotDescriptor(const PSlotDescriptor& pdesc);
MOCK_DEFINE(SlotDescriptor();)
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index bd996f2189b..cfbad191772 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -26,6 +26,7 @@
#include <algorithm>
#include <cctype>
+#include <list>
#include "vec/exprs/vdirect_in_predicate.h"
#include "vec/exprs/vexpr.h"
@@ -355,8 +356,11 @@ Status OrcReader::init_reader(
const RowDescriptor* row_descriptor,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
- std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr,
+ const std::set<uint64_t>& column_ids, const std::set<uint64_t>& filter_column_ids) {
_table_column_names = column_names;
+ _column_ids = column_ids;
+ _filter_column_ids = filter_column_ids;
_colname_to_value_range = colname_to_value_range;
_lazy_read_ctx.conjuncts = conjuncts;
_is_acid = is_acid;
@@ -401,6 +405,32 @@ Status OrcReader::get_parsed_schema(std::vector<std::string>* col_names,
Status OrcReader::_init_read_columns() {
SCOPED_RAW_TIMER(&_statistics.init_column_time);
const auto& root_type = _reader->getType();
+
+ // Save file root type for later use in type conversion
+ _file_root_type = &root_type;
+
+ // Build column ID to file type mapping for all columns in file
+ _column_id_to_file_type.clear();
+ std::function<void(const orc::Type*, int)> build_column_id_map = [&](const orc::Type* type,
+ int depth) {
+ if (type == nullptr) return;
+ uint64_t col_id = type->getColumnId();
+ _column_id_to_file_type[col_id] = type;
+
+ std::string indent(depth * 2, ' ');
+ VLOG_DEBUG << indent << "[OrcReader] Mapping column_id=" << col_id
+ << ", kind=" << static_cast<int>(type->getKind())
+ << ", subtype_count=" << type->getSubtypeCount();
+
+ for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
+ build_column_id_map(type->getSubtype(i), depth + 1);
+ }
+ };
+
+ VLOG_DEBUG << "[OrcReader] Building column ID to file type mapping...";
+ build_column_id_map(&root_type, 0);
+ VLOG_DEBUG << "[OrcReader] Total mapped columns: " << _column_id_to_file_type.size();
+
if (_is_acid) {
for (uint64_t i = 0; i < root_type.getSubtypeCount(); ++i) {
if (root_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) {
@@ -650,7 +680,7 @@ std::pair<bool, orc::Literal> OrcReader::_make_orc_literal(const VSlotRef* slot_
auto* slot = _tuple_descriptor->slots()[slot_ref->column_id()];
auto slot_type = slot->type();
auto primitive_type = slot_type->get_primitive_type();
- auto src_type = OrcReader::convert_to_doris_type(orc_type)->get_primitive_type();
+ auto src_type = convert_to_doris_type(orc_type)->get_primitive_type();
// should not down predicate for string type change from other type
if (src_type != primitive_type && !is_string_type(src_type) && is_string_type(primitive_type)) {
LOG(WARNING) << "Unsupported Push Down Schema Changed Column " << primitive_type << " to "
@@ -1158,7 +1188,12 @@ Status OrcReader::set_fill_columns(
try {
_row_reader_options.range(_range_start_offset, _range_size);
_row_reader_options.setTimezoneName(_ctz == "CST" ? "Asia/Shanghai" : _ctz);
- _row_reader_options.include(_read_file_cols);
+ if (!_column_ids.empty()) {
+ std::list<uint64_t> column_ids_list(_column_ids.begin(), _column_ids.end());
+ _row_reader_options.includeTypes(column_ids_list);
+ } else {
+ _row_reader_options.include(_read_file_cols);
+ }
_row_reader_options.setEnableLazyDecoding(true);
//orc reader should not use the tiny stripe optimization when reading by row id.
@@ -1246,6 +1281,57 @@ Status OrcReader::set_fill_columns(
}
}
+ // Helper function to recursively print ORC type structure
+ std::function<void(const orc::Type*, int, const std::string&)> print_orc_type_tree =
+ [&](const orc::Type* type, int indent, const std::string& name) {
+ if (type) {
+ std::string indent_str(indent * 2, ' ');
+ VLOG_DEBUG << indent_str << "- " << name
+ << " (type_id=" << type->getColumnId()
+ << ", kind=" << static_cast<int>(type->getKind()) << ")";
+ }
+
+ if (type->getKind() == orc::TypeKind::STRUCT) {
+ for (uint64_t i = 0; i < type->getSubtypeCount(); ++i) {
+ std::string field_name = type->getFieldName(i);
+ print_orc_type_tree(type->getSubtype(i), indent + 1, field_name);
+ }
+ } else if (type->getKind() == orc::TypeKind::LIST) {
+ print_orc_type_tree(type->getSubtype(0), indent + 1, "element");
+ } else if (type->getKind() == orc::TypeKind::MAP) {
+ print_orc_type_tree(type->getSubtype(0), indent + 1, "key");
+ print_orc_type_tree(type->getSubtype(1), indent + 1, "value");
+ } else {
+ // Primitive type, no children
+ }
+ };
+
+ // VLOG_DEBUG << "\n[OrcReader] ========== ORC Type Structure ==========";
+ // print_orc_type_tree(&selected_type, 0, "root");
+ // VLOG_DEBUG << "[OrcReader] ==========================================\n";
+
+ _type_map.clear();
+ if (_is_acid) {
+ for (uint64_t i = 0; i < selected_type.getSubtypeCount(); ++i) {
+ if (selected_type.getSubtype(i)->getKind() == orc::TypeKind::STRUCT) {
+ auto row_orc_type = selected_type.getSubtype(i);
+ for (uint64_t j = 0; j < row_orc_type->getSubtypeCount(); j++) {
+ std::string field_name =
+ TransactionalHive::ROW + "." + row_orc_type->getFieldName(j);
+ _type_map.emplace(field_name, row_orc_type->getSubtype(j));
+ }
+ } else {
+ std::string field_name = selected_type.getFieldName(i);
+ _type_map.emplace(field_name, selected_type.getSubtype(i));
+ }
+ }
+ } else {
+ for (int i = 0; i < selected_type.getSubtypeCount(); ++i) {
+ std::string field_name = selected_type.getFieldName(i);
+ _type_map.emplace(field_name, selected_type.getSubtype(i));
+ }
+ }
+
_remaining_rows = _row_reader->getNumberOfRows();
} catch (std::exception& e) {
@@ -1343,6 +1429,11 @@ Status OrcReader::_fill_missing_columns(
block->erase(result_column_id);
}
}
+
+ // // no default column, fill with null
+ // auto mutable_column = block->get_by_name(kv.first).column->assume_mutable();
+ // auto* nullable_column = static_cast<vectorized::ColumnNullable*>(mutable_column.get());
+ // nullable_column->insert_many_defaults(rows);
}
return Status::OK();
}
@@ -1391,6 +1482,14 @@ void OrcReader::_init_file_description() {
}
DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
+ // Critical: check for nullptr BEFORE accessing any methods
+ if (orc_type == nullptr) {
+ LOG(WARNING) << "[OrcReader] ERROR: convert_to_doris_type called with nullptr orc_type! "
+ "Falling back to STRING";
+ // Return a safe default type instead of crashing
+ return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_STRING, true);
+ }
+
switch (orc_type->getKind()) {
case orc::TypeKind::BOOLEAN:
return DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_BOOLEAN, true);
@@ -1437,6 +1536,74 @@ DataTypePtr OrcReader::convert_to_doris_type(const orc::Type* orc_type) {
std::make_shared<DataTypeArray>(convert_to_doris_type(orc_type->getSubtype(0))));
}
case orc::TypeKind::MAP: {
+ // Handle incomplete MAP type due to column pruning
+ // If MAP doesn't have both key and value subtypes, try to find the complete type from file
+ if (orc_type->getSubtypeCount() < 2 || orc_type->getSubtype(0) == nullptr ||
+ orc_type->getSubtype(1) == nullptr) {
+ // Try to find the complete type from _column_id_to_file_type
+ uint64_t column_id = orc_type->getColumnId();
+ VLOG(1) << "[OrcReader] Detected incomplete MAP type: column_id=" << column_id
+ << ", subtype_count=" << orc_type->getSubtypeCount() << ", subtype(0)="
+ << (orc_type->getSubtypeCount() > 0 && orc_type->getSubtype(0) != nullptr
+ ? "not null"
+ : "null")
+ << ", subtype(1)="
+ << (orc_type->getSubtypeCount() > 1 && orc_type->getSubtype(1) != nullptr
+ ? "not null"
+ : "null");
+
+ auto it = _column_id_to_file_type.find(column_id);
+ if (it != _column_id_to_file_type.end() && it->second != nullptr) {
+ const orc::Type* complete_type = it->second;
+ VLOG(1) << "[OrcReader] Found complete type in mapping: column_id=" << column_id
+ << ", complete_type_kind=" << static_cast<int>(complete_type->getKind())
+ << ", complete_subtype_count=" << complete_type->getSubtypeCount();
+
+ // Print subtypes information
+ for (uint64_t i = 0; i < complete_type->getSubtypeCount(); ++i) {
+ const orc::Type* subtype = complete_type->getSubtype(i);
+ VLOG_DEBUG << "[OrcReader] complete_type->subtype[" << i << "]: "
+ << (subtype != nullptr
+ ? ("kind=" +
+ std::to_string(static_cast<int>(subtype->getKind())) +
+ ", column_id=" +
+ std::to_string(subtype->getColumnId()))
+ : "nullptr");
+ }
+
+ if (complete_type->getKind() == orc::TypeKind::MAP &&
+ complete_type->getSubtypeCount() == 2) {
+ VLOG(1) << "[OrcReader] Using complete MAP type from file schema for column_id="
+ << column_id;
+
+ // Get subtypes with extra validation
+ const orc::Type* key_type = complete_type->getSubtype(0);
+ const orc::Type* value_type = complete_type->getSubtype(1);
+
+ VLOG_DEBUG << "[OrcReader] About to convert key_type: "
+ << (key_type != nullptr ? "not null" : "NULL");
+ VLOG_DEBUG << "[OrcReader] About to convert value_type: "
+ << (value_type != nullptr ? "not null" : "NULL");
+
+ // Use the complete type from file - with null checks
+ DataTypePtr key_doris_type = convert_to_doris_type(key_type);
+ VLOG_DEBUG << "[OrcReader] Successfully converted key_type";
+
+ DataTypePtr value_doris_type = convert_to_doris_type(value_type);
+ VLOG_DEBUG << "[OrcReader] Successfully converted value_type";
+
+ return make_nullable(
+ std::make_shared<DataTypeMap>(key_doris_type, value_doris_type));
+ } else {
+ LOG(WARNING) << "[OrcReader] Warning: Complete type is not a valid MAP or has "
+ "wrong subtype count";
+ }
+ } else {
+ LOG(WARNING) << "[OrcReader] Warning: Could not find complete type in mapping for "
+ "column_id="
+ << column_id << ", mapping_size=" << _column_id_to_file_type.size();
+ }
+ }
return make_nullable(
std::make_shared<DataTypeMap>(convert_to_doris_type(orc_type->getSubtype(0)),
convert_to_doris_type(orc_type->getSubtype(1))));
@@ -1682,6 +1849,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const orc::Type* orc_column_type,
const orc::ColumnVectorBatch* cvb, size_t num_values) {
auto logical_type = data_type->get_primitive_type();
+
switch (logical_type) {
#define DISPATCH(FlatType, CppType, OrcColumnType) \
case FlatType: \
@@ -1752,21 +1920,99 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const DataTypePtr& doris_value_type =
reinterpret_cast<const DataTypeMap*>(remove_nullable(data_type).get())
->get_value_type();
+
+ // Get ORC key and value types with null checks
const orc::Type* orc_key_type = orc_column_type->getSubtype(0);
const orc::Type* orc_value_type = orc_column_type->getSubtype(1);
+
+ VLOG(1) << "[OrcReader] MAP column '" << col_name
+ << "': orc_key_type=" << (orc_key_type != nullptr ? "not null" : "NULL")
+ << ", orc_value_type=" << (orc_value_type != nullptr ? "not null" : "NULL")
+ << ", element_size=" << element_size;
+
+ // Handle incomplete MAP type - if key or value type is nullptr, try to recover from mapping
+ bool key_is_missing = (orc_key_type == nullptr);
+ bool value_is_missing = (orc_value_type == nullptr);
+
+ if (key_is_missing || value_is_missing) {
+ VLOG(1) << "[OrcReader] Detected incomplete MAP subtypes for column '" << col_name
+ << "', attempting to recover from mapping...";
+
+ uint64_t column_id = orc_column_type->getColumnId();
+ auto it = _column_id_to_file_type.find(column_id);
+ if (it != _column_id_to_file_type.end() && it->second != nullptr) {
+ const orc::Type* complete_map_type = it->second;
+ if (complete_map_type->getKind() == orc::TypeKind::MAP &&
+ complete_map_type->getSubtypeCount() == 2) {
+ if (key_is_missing) {
+ orc_key_type = complete_map_type->getSubtype(0);
+ if (orc_key_type != nullptr) {
+ // key_is_missing = false;
+ VLOG(1) << "[OrcReader] Recovered key type from mapping for column '"
+ << col_name << "'";
+ }
+ }
+ if (value_is_missing) {
+ orc_value_type = complete_map_type->getSubtype(1);
+ if (orc_value_type != nullptr) {
+ // value_is_missing = false;
+ VLOG(1) << "[OrcReader] Recovered value type from mapping for column '"
+ << col_name << "'";
+ }
+ }
+ }
+ }
+ }
+
ColumnPtr& doris_key_column = doris_map.get_keys_ptr();
ColumnPtr& doris_value_column = doris_map.get_values_ptr();
std::string key_col_name = col_name + ".key";
std::string value_col_name = col_name + ".value";
- RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
- key_col_name, doris_key_column, doris_key_type, root_node->get_key_node(),
+ // Handle key column: if still missing, fill with default values
+ if (key_is_missing) {
+ LOG(INFO) << "[OrcReader] Key type is missing for MAP column '" << col_name
+ << "', filling with default values (element_size=" << element_size << ")";
+
+ // Fill key column with default values (nulls or empty values)
+ auto mutable_key_column = doris_key_column->assume_mutable();
+ if (mutable_key_column->is_nullable()) {
+ auto* nullable_column = static_cast<ColumnNullable*>(mutable_key_column.get());
+ nullable_column->insert_many_defaults(element_size);
+ } else {
+ mutable_key_column->insert_many_defaults(element_size);
+ }
+ } else {
+ // Normal processing: convert ORC column to Doris column
+ auto status = _orc_column_to_doris_column<false>(
+ key_col_name, doris_key_column, doris_key_type, root_node->get_key_node(),
+ orc_key_type, orc_map->keys.get(), element_size);
+ if (!status.ok()) {
+ return status;
+ }
+ }
+
+ // Handle value column: if still missing, fill with default values
+ if (value_is_missing) {
+ LOG(INFO) << "[OrcReader] Value type is missing for MAP column '" << col_name
+ << "', filling with default values (element_size=" << element_size << ")";
- orc_key_type, orc_map->keys.get(), element_size));
- RETURN_IF_ERROR(_orc_column_to_doris_column<false>(
- value_col_name, doris_value_column, doris_value_type, root_node->get_value_node(),
- orc_value_type, orc_map->elements.get(), element_size));
- return doris_map.deduplicate_keys();
+ // Fill value column with default values (nulls or empty values)
+ auto mutable_value_column = doris_value_column->assume_mutable();
+ if (mutable_value_column->is_nullable()) {
+ auto* nullable_column = static_cast<ColumnNullable*>(mutable_value_column.get());
+ nullable_column->insert_many_defaults(element_size);
+ } else {
+ mutable_value_column->insert_many_defaults(element_size);
+ }
+ return Status::OK();
+ } else {
+ // Normal processing: convert ORC column to Doris column
+ return _orc_column_to_doris_column<false>(value_col_name, doris_value_column,
+ doris_value_type, root_node->get_value_node(),
+ orc_value_type, orc_map->elements.get(),
+ element_size);
+ }
}
case PrimitiveType::TYPE_STRUCT: {
if (orc_column_type->getKind() != orc::TypeKind::STRUCT) {
@@ -1781,6 +2027,14 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
const auto* doris_struct_type =
assert_cast<const DataTypeStruct*>(remove_nullable(data_type).get());
+ // Build ORC field name to index map for faster lookup
+ std::unordered_map<std::string, int> orc_field_name_to_idx;
+ for (int j = 0; j < orc_column_type->getSubtypeCount(); ++j) {
+ std::string field_name = orc_column_type->getFieldName(j);
+ std::transform(field_name.begin(), field_name.end(), field_name.begin(), ::tolower);
+ orc_field_name_to_idx[field_name] = j;
+ }
+
for (int i = 0; i < doris_struct.tuple_size(); ++i) {
const auto& table_column_name = doris_struct_type->get_name_by_position(i);
if (!root_node->children_column_exists(table_column_name)) {
@@ -1788,10 +2042,23 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
continue;
}
const auto& file_column_name = root_node->children_file_column_name(table_column_name);
- for (int j = 0; j < orc_column_type->getSubtypeCount(); ++j) {
- if (boost::iequals(orc_column_type->getFieldName(j), file_column_name)) {
- read_fields[i] = j;
- }
+ std::string file_column_name_lower = file_column_name;
+ std::transform(file_column_name_lower.begin(), file_column_name_lower.end(),
+ file_column_name_lower.begin(), ::tolower);
+
+ auto it = orc_field_name_to_idx.find(file_column_name_lower);
+ if (it != orc_field_name_to_idx.end()) {
+ read_fields[i] = it->second;
+ VLOG_DEBUG << "[OrcReader] Found field mapping: doris_field[" << i
+ << "] -> orc_field[" << it->second
+ << "], table_column: " << table_column_name
+ << ", file_column: " << file_column_name_lower;
+ } else {
+ missing_fields.insert(i);
+ VLOG_DEBUG << "[OrcReader] Missing field: doris_field[" << i
+ << "], table_column: " << table_column_name
+ << ", file_column: " << file_column_name_lower
+ << " (not found in ORC file)";
}
}
@@ -1808,6 +2075,7 @@ Status OrcReader::_fill_doris_data_column(const std::string& col_name,
for (auto read_field : read_fields) {
orc::ColumnVectorBatch* orc_field = orc_struct->fields[read_field.second];
+
const orc::Type* orc_type = orc_column_type->getSubtype(read_field.second);
std::string field_name =
col_name + "." + orc_column_type->getFieldName(read_field.second);
@@ -1832,67 +2100,81 @@ Status OrcReader::_orc_column_to_doris_column(
const std::string& col_name, ColumnPtr& doris_column, const DataTypePtr& data_type,
std::shared_ptr<TableSchemaChangeHelper::Node> root_node, const orc::Type* orc_column_type,
const orc::ColumnVectorBatch* cvb, size_t num_values) {
- auto src_type = convert_to_doris_type(orc_column_type);
- bool is_dict_filter_col = false;
- for (const std::pair<std::string, int>& dict_col : _dict_filter_cols) {
- if (col_name == dict_col.first) {
- src_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true);
- is_dict_filter_col = true;
- break;
- }
- }
- // If the column can be dictionary filtered, there will be two types.
- // It may be plain or a dictionary, because the same field in different stripes may have different types.
- // Here we use the $dict_ prefix to represent the dictionary type converter.
- auto converter_key = !is_dict_filter_col ? col_name : fmt::format("$dict_{}", col_name);
-
- if (!_converters.contains(converter_key)) {
- std::unique_ptr<converter::ColumnTypeConverter> converter =
- converter::ColumnTypeConverter::get_converter(src_type, data_type,
- converter::FileFormat::ORC);
- if (!converter->support()) {
- return Status::InternalError(
- "The column type of '{}' has changed and is not supported: ", col_name,
- converter->get_error_msg());
- }
- // reuse the cached converter
- _converters[converter_key] = std::move(converter);
- }
- converter::ColumnTypeConverter* converter = _converters[converter_key].get();
- ColumnPtr resolved_column = converter->get_column(src_type, doris_column, data_type);
- const DataTypePtr& resolved_type = converter->get_type();
-
+ DataTypePtr resolved_type;
+ ColumnPtr resolved_column;
+ converter::ColumnTypeConverter* converter_ptr = nullptr;
MutableColumnPtr data_column;
- if (resolved_column->is_nullable()) {
- SCOPED_RAW_TIMER(&_statistics.decode_null_map_time);
- auto* nullable_column =
- reinterpret_cast<ColumnNullable*>(resolved_column->assume_mutable().get());
- data_column = nullable_column->get_nested_column_ptr();
- NullMap& map_data_column = nullable_column->get_null_map_data();
- auto origin_size = map_data_column.size();
- map_data_column.resize(origin_size + num_values);
- if (cvb->hasNulls) {
- const auto* cvb_nulls = cvb->notNull.data();
- for (int i = 0; i < num_values; ++i) {
- map_data_column[origin_size + i] = !cvb_nulls[i];
+ if (orc_column_type != nullptr) {
+ auto src_type = convert_to_doris_type(orc_column_type);
+ bool is_dict_filter_col = false;
+ for (const std::pair<std::string, int>& dict_col : _dict_filter_cols) {
+ if (col_name == dict_col.first) {
+ src_type =
+ DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_INT, true);
+ is_dict_filter_col = true;
+ break;
+ }
+ }
+ // If the column can be dictionary filtered, there will be two types.
+ // It may be plain or a dictionary, because the same field in different stripes may have different types.
+ // Here we use the $dict_ prefix to represent the dictionary type converter.
+ auto converter_key = !is_dict_filter_col ? col_name : fmt::format("$dict_{}", col_name);
+
+ if (!_converters.contains(converter_key)) {
+ std::unique_ptr<converter::ColumnTypeConverter> converter =
+ converter::ColumnTypeConverter::get_converter(src_type, data_type,
+ converter::FileFormat::ORC);
+ if (!converter->support()) {
+ return Status::InternalError(
+ "The column type of '{}' has changed and is not supported: ", col_name,
+ converter->get_error_msg());
+ }
+ // reuse the cached converter
+ _converters[converter_key] = std::move(converter);
+ }
+ converter_ptr = _converters[converter_key].get();
+ resolved_column = converter_ptr->get_column(src_type, doris_column, data_type);
+ resolved_type = converter_ptr->get_type();
+
+ if (resolved_column->is_nullable()) {
+ SCOPED_RAW_TIMER(&_statistics.decode_null_map_time);
+ auto* nullable_column =
+ reinterpret_cast<ColumnNullable*>(resolved_column->assume_mutable().get());
+ data_column = nullable_column->get_nested_column_ptr();
+
+ NullMap& map_data_column = nullable_column->get_null_map_data();
+ auto origin_size = map_data_column.size();
+ map_data_column.resize(origin_size + num_values);
+ if (cvb->hasNulls) {
+ const auto* cvb_nulls = cvb->notNull.data();
+ for (int i = 0; i < num_values; ++i) {
+ map_data_column[origin_size + i] = !cvb_nulls[i];
+ }
+ } else {
+ memset(map_data_column.data() + origin_size, 0, num_values);
}
} else {
- memset(map_data_column.data() + origin_size, 0, num_values);
+ if (cvb->hasNulls) {
+ return Status::InternalError("Not nullable column {} has null values in orc file",
+ col_name);
+ }
+ data_column = resolved_column->assume_mutable();
}
+
+ RETURN_IF_ERROR((_fill_doris_data_column<is_filter>(
+ col_name, data_column, remove_nullable(resolved_type), root_node, orc_column_type,
+ cvb, num_values)));
+ // if (converter_ptr != nullptr) {
+ // resolve schema change
+ auto converted_column = doris_column->assume_mutable();
+ return converter_ptr->convert(resolved_column, converted_column);
+ // }
} else {
- if (cvb->hasNulls) {
- return Status::InternalError("Not nullable column {} has null values in orc file",
- col_name);
- }
- data_column = resolved_column->assume_mutable();
+ reinterpret_cast<ColumnNullable*>(doris_column->assume_mutable().get())
+ ->insert_many_defaults(num_values);
}
- RETURN_IF_ERROR(_fill_doris_data_column<is_filter>(col_name, data_column,
- remove_nullable(resolved_type), root_node,
- orc_column_type, cvb, num_values));
- // resolve schema change
- auto converted_column = doris_column->assume_mutable();
- return converter->convert(resolved_column, converted_column);
+ return Status::OK();
}
std::string OrcReader::get_field_name_lower_case(const orc::Type* orc_type, int pos) {
@@ -1993,10 +2275,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column<true>(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<true>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements));
+ batch_vec[orc_col_idx->second], _batch->numElements)));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2051,6 +2333,12 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
}
}
+ //std::cout << "[OrcReader] _column_ids: ";
+ //for (const auto& cid : _column_ids) {
+ // std::cout << cid << " ";
+ //}
+ //std::cout << std::endl;
+
if (!_dict_cols_has_converted && !_dict_filter_cols.empty()) {
for (auto& dict_filter_cols : _dict_filter_cols) {
MutableColumnPtr dict_col_ptr = ColumnInt32::create();
@@ -2084,10 +2372,10 @@ Status OrcReader::_get_next_block_impl(Block* block, size_t* read_rows, bool* eo
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(col_name), _type_map[file_column_name],
- batch_vec[orc_col_idx->second], _batch->numElements));
+ batch_vec[orc_col_idx->second], _batch->numElements)));
}
RETURN_IF_ERROR(_fill_partition_columns(block, _batch->numElements,
@@ -2250,10 +2538,10 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s
if (orc_col_idx == _colname_to_idx.end()) {
return Status::InternalError("Wrong read column '{}' in orc file", table_col_name);
}
- RETURN_IF_ERROR(_orc_column_to_doris_column(
+ RETURN_IF_ERROR((_orc_column_to_doris_column<false>(
table_col_name, column_ptr, column_type,
_table_info_node_ptr->get_children_node(table_col_name),
- _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements));
+ _type_map[file_column_name], batch_vec[orc_col_idx->second], data.numElements)));
}
RETURN_IF_ERROR(
_fill_partition_columns(block, size, _lazy_read_ctx.predicate_partition_columns));
diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h
index 358ba1d2468..341e40a9ce2 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.h
+++ b/be/src/vec/exec/format/orc/vorc_reader.h
@@ -112,6 +112,12 @@ struct LazyReadContext {
std::unordered_map<std::string, VExprContextSPtr> predicate_missing_columns;
// lazy read missing columns or all missing columns
std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+
+ std::vector<std::string> partial_predicate_columns;
+
+ // Record the number of rows filled in filter phase for lazy materialization
+ // This is used to check if a column was already processed in filter phase
+ size_t filter_phase_rows = 0;
};
class OrcReader : public GenericReader {
@@ -158,7 +164,9 @@ public:
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr =
- TableSchemaChangeHelper::ConstNode::get_instance());
+ TableSchemaChangeHelper::ConstNode::get_instance(),
+ const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
Status set_fill_columns(
const std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>&
@@ -195,7 +203,8 @@ public:
std::unordered_map<std::string, orc::StringDictionary*>& column_name_to_dict_map,
bool* is_stripe_filtered);
- static DataTypePtr convert_to_doris_type(const orc::Type* orc_type);
+ DataTypePtr convert_to_doris_type(const orc::Type* orc_type);
+
static std::string get_field_name_lower_case(const orc::Type* orc_type, int pos);
void set_row_id_column_iterator(
@@ -630,7 +639,8 @@ private:
size_t _batch_size;
int64_t _range_start_offset;
int64_t _range_size;
- const std::string& _ctz;
+ // const std::string& _ctz;
+ std::string _ctz;
int32_t _offset_days = 0;
cctz::time_zone _time_zone;
@@ -653,6 +663,12 @@ private:
// file column name to orc type
std::unordered_map<std::string, const orc::Type*> _type_map;
+ // Optimized type conversion support
+ // column ID to file original type mapping for handling partial column selection
+ std::unordered_map<uint64_t, const orc::Type*> _column_id_to_file_type;
+ // Keep reference to file root type for complete type information
+ const orc::Type* _file_root_type = nullptr;
+
std::unique_ptr<ORCFileInputStream> _file_input_stream;
Statistics _statistics;
OrcProfile _orc_profile;
@@ -673,6 +689,8 @@ private:
std::vector<DecimalScaleParams> _decimal_scale_params;
size_t _decimal_scale_params_index;
+ std::set<uint64_t> _column_ids;
+ std::set<uint64_t> _filter_column_ids;
const std::unordered_map<std::string, ColumnValueRangeType>* _colname_to_value_range = nullptr;
bool _is_acid = false;
std::unique_ptr<IColumn::Filter> _filter;
diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp
index 82e6e0c9d61..c022f1e3cfe 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.cpp
+++ b/be/src/vec/exec/format/parquet/schema_desc.cpp
@@ -176,6 +176,7 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
parse_physical_field(t_schema, false, child);
node_field->name = t_schema.name;
+ node_field->lower_case_name = to_lower(t_schema.name);
node_field->data_type = std::make_shared<DataTypeArray>(make_nullable(child->data_type));
_next_schema_pos = curr_pos + 1;
node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1;
@@ -193,8 +194,10 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
void FieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema,
bool is_nullable, FieldSchema* physical_field) {
physical_field->name = physical_schema.name;
+ physical_field->lower_case_name = to_lower(physical_field->name);
physical_field->parquet_schema = physical_schema;
physical_field->physical_type = physical_schema.type;
+ physical_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
_physical_fields.push_back(physical_field);
physical_field->physical_column_index = cast_set<int>(_physical_fields.size() - 1);
auto type = get_doris_type(physical_schema, is_nullable);
@@ -393,6 +396,8 @@ Status FieldDescriptor::parse_group_field(const std::vector<tparquet::SchemaElem
RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field));
group_field->name = group_schema.name;
+ group_field->lower_case_name = to_lower(group_field->name);
+ group_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
group_field->data_type =
std::make_shared<DataTypeArray>(make_nullable(struct_field->data_type));
group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1;
@@ -461,6 +466,8 @@ Status FieldDescriptor::parse_list_field(const std::vector<tparquet::SchemaEleme
}
list_field->name = first_level.name;
+ list_field->lower_case_name = to_lower(first_level.name);
+ list_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
list_field->data_type =
std::make_shared<DataTypeArray>(make_nullable(list_field->children[0].data_type));
if (is_optional) {
@@ -519,21 +526,24 @@ Status FieldDescriptor::parse_map_field(const std::vector<tparquet::SchemaElemen
map_field->repetition_level++;
map_field->definition_level++;
- map_field->children.resize(1);
+ // Directly create key and value children instead of intermediate key_value node
+ map_field->children.resize(2);
// map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level`
set_child_node_level(map_field, map_field->definition_level);
- auto map_kv_field = &map_field->children[0];
- // produce MAP<STRUCT<KEY, VALUE>>
- RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, map_kv_field));
+
+ auto key_field = &map_field->children[0];
+ auto value_field = &map_field->children[1];
+
+ // Parse key and value fields directly from the key_value group's children
+ _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key
+ RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field));
+ RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field));
map_field->name = map_schema.name;
- map_field->data_type = std::make_shared<DataTypeMap>(
- make_nullable(assert_cast<const DataTypeStruct*>(
- remove_nullable(map_kv_field->data_type).get())
- ->get_element(0)),
- make_nullable(assert_cast<const DataTypeStruct*>(
- remove_nullable(map_kv_field->data_type).get())
- ->get_element(1)));
+ map_field->lower_case_name = to_lower(map_field->name);
+ map_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
+ map_field->data_type = std::make_shared<DataTypeMap>(make_nullable(key_field->data_type),
+ make_nullable(value_field->data_type));
if (is_optional) {
map_field->data_type = make_nullable(map_field->data_type);
}
@@ -558,6 +568,8 @@ Status FieldDescriptor::parse_struct_field(const std::vector<tparquet::SchemaEle
RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i]));
}
struct_field->name = struct_schema.name;
+ struct_field->lower_case_name = to_lower(struct_field->name);
+ struct_field->column_id = UNASSIGNED_COLUMN_ID; // Initialize column_id
struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1;
DataTypes res_data_types;
@@ -610,6 +622,59 @@ std::string FieldDescriptor::debug_string() const {
ss << "]";
return ss.str();
}
+
+void FieldDescriptor::assign_ids() {
+ uint64_t next_id = 1;
+ for (auto& field : _fields) {
+ field.assign_ids(next_id);
+ }
+}
+
+const FieldSchema* FieldDescriptor::find_column_by_id(uint64_t column_id) const {
+ for (const auto& field : _fields) {
+ if (auto result = field.find_column_by_id(column_id)) {
+ return result;
+ }
+ }
+ return nullptr;
+}
+
+void FieldSchema::assign_ids(uint64_t& next_id) {
+ column_id = next_id++;
+
+ for (auto& child : children) {
+ child.assign_ids(next_id);
+ }
+
+ max_column_id = next_id - 1;
+}
+
+const FieldSchema* FieldSchema::find_column_by_id(uint64_t target_id) const {
+ if (column_id == target_id) {
+ return this;
+ }
+
+ for (const auto& child : children) {
+ if (auto result = child.find_column_by_id(target_id)) {
+ return result;
+ }
+ }
+
+ return nullptr;
+}
+
+uint64_t FieldSchema::get_column_id() const {
+ return column_id;
+}
+
+void FieldSchema::set_column_id(uint64_t id) {
+ column_id = id;
+}
+
+uint64_t FieldSchema::get_max_column_id() const {
+ return max_column_id;
+}
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/schema_desc.h b/be/src/vec/exec/format/parquet/schema_desc.h
index 09a52a4251a..3ba38071406 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.h
+++ b/be/src/vec/exec/format/parquet/schema_desc.h
@@ -36,8 +36,13 @@
namespace doris::vectorized {
#include "common/compile_check_begin.h"
+
+// Constant for unassigned column IDs
+constexpr uint64_t UNASSIGNED_COLUMN_ID = UINT64_MAX;
+
struct FieldSchema {
std::string name;
+ std::string lower_case_name; // for hms column name case insensitive match
// the referenced parquet schema element
tparquet::SchemaElement parquet_schema;
@@ -56,12 +61,22 @@ struct FieldSchema {
//For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128.
bool is_type_compatibility = false;
- FieldSchema() : data_type(std::make_shared<DataTypeNothing>()) {}
+ FieldSchema()
+ : data_type(std::make_shared<DataTypeNothing>()), column_id(UNASSIGNED_COLUMN_ID) {}
~FieldSchema() = default;
FieldSchema(const FieldSchema& fieldSchema) = default;
std::string debug_string() const;
int32_t field_id = -1;
+ uint64_t column_id = UNASSIGNED_COLUMN_ID;
+ uint64_t max_column_id = 0; // Maximum column ID for this field and its children
+
+ // Column ID assignment and lookup methods
+ void assign_ids(uint64_t& next_id);
+ const FieldSchema* find_column_by_id(uint64_t target_id) const;
+ uint64_t get_column_id() const;
+ void set_column_id(uint64_t id);
+ uint64_t get_max_column_id() const;
};
class FieldDescriptor {
@@ -133,6 +148,19 @@ public:
int32_t size() const { return cast_set<int32_t>(_fields.size()); }
const std::vector<FieldSchema>& get_fields_schema() const { return _fields; }
+
+ /**
+ * Assign stable column IDs to schema fields.
+ *
+ * This uses an ORC-compatible encoding so that the results of
+ * create_column_ids() are consistent across formats. IDs start from 1
+ * and are assigned in a pre-order traversal (parent before children).
+ * After calling this, each FieldSchema will have column_id and
+ * max_column_id populated.
+ */
+ void assign_ids();
+
+ const FieldSchema* find_column_by_id(uint64_t column_id) const;
};
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index 74fe4c3601b..a7455455198 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -108,45 +108,120 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field,
const std::vector<RowRange>& row_ranges,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
std::unique_ptr<ParquetColumnReader>& reader,
- size_t max_buf_size, const tparquet::OffsetIndex* offset_index) {
+ size_t max_buf_size, const tparquet::OffsetIndex* offset_index,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids) {
+ // if (!column_ids.empty()) {
+ // uint64_t field_column_id = field->get_column_id();
+ // VLOG_DEBUG << "[ParquetReader] Checking column_id: " << field_column_id
+ // << ", column_ids.size(): " << column_ids.size();
+ // if (column_ids.find(field_column_id) == column_ids.end()) {
+ // VLOG_DEBUG << "[ParquetReader] Setting skip_reading=true for column_id: "
+ // << field_column_id;
+ // auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field);
+ // skip_reader->_filter_column_ids = filter_column_ids;
+ // reader = std::move(skip_reader);
+ // return Status::OK();
+ // } else {
+ // VLOG_DEBUG << "[ParquetReader] Column_id " << field_column_id
+ // << " found in column_ids, skip_reading=false";
+ // }
+ // }
+
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
std::unique_ptr<ParquetColumnReader> element_reader;
RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
- element_reader, max_buf_size));
+ element_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
element_reader->set_nested_column();
auto array_reader = ArrayColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field));
+ array_reader->_filter_column_ids = filter_column_ids;
reader.reset(array_reader.release());
} else if (field->data_type->get_primitive_type() == TYPE_MAP) {
std::unique_ptr<ParquetColumnReader> key_reader;
std::unique_ptr<ParquetColumnReader> value_reader;
- RETURN_IF_ERROR(create(file, &field->children[0].children[0], row_group, row_ranges, ctz,
- io_ctx, key_reader, max_buf_size));
- RETURN_IF_ERROR(create(file, &field->children[0].children[1], row_group, row_ranges, ctz,
- io_ctx, value_reader, max_buf_size));
- key_reader->set_nested_column();
- value_reader->set_nested_column();
+
+ if (column_ids.empty() ||
+ column_ids.find(field->children[0].get_column_id()) != column_ids.end()) {
+ // Create key reader
+ RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
+ key_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
+ key_reader->set_nested_column();
+ } else {
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx,
+ &field->children[0]);
+ key_reader = std::move(skip_reader);
+ }
+
+ if (column_ids.empty() ||
+ column_ids.find(field->children[1].get_column_id()) != column_ids.end()) {
+ // Create value reader
+ RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx,
+ value_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
+ value_reader->set_nested_column();
+ } else {
+ auto skip_reader = std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx,
+ &field->children[0]);
+ value_reader = std::move(skip_reader);
+ }
+
auto map_reader = MapColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field));
+ map_reader->_filter_column_ids = filter_column_ids;
reader.reset(map_reader.release());
} else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
std::unordered_map<std::string, std::unique_ptr<ParquetColumnReader>> child_readers;
child_readers.reserve(field->children.size());
+ int non_skip_reader_idx = -1;
for (int i = 0; i < field->children.size(); ++i) {
+ auto& child = field->children[i]; // 去掉 const，保证 &child 类型为 FieldSchema*
+ std::unique_ptr<ParquetColumnReader> child_reader;
+ if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) {
+ RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx,
+ child_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
+ child_reader->set_nested_column();
+ child_readers[child.name] = std::move(child_reader);
+ // 记录第一个非 SkippingReader
+ if (non_skip_reader_idx == -1) {
+ non_skip_reader_idx = i;
+ }
+ } else {
+ // 不在 column_ids 的 child，创建 SkippingReader
+ auto skip_reader =
+ std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, &child);
+ skip_reader->_filter_column_ids = filter_column_ids;
+ child_readers[child.name] = std::move(skip_reader);
+ }
+ }
+ // 如果所有都是 SkipReadingReader，则强制第一个 child 调用 create
+ if (non_skip_reader_idx == -1) {
+ // for (int i = 0; i < field->children.size(); ++i) {
+ // 只处理 SkipReadingReader（即刚才创建的）
+ // if (dynamic_cast<SkipReadingReader*>(child_readers[field->children[i].name].get()) != nullptr) {
std::unique_ptr<ParquetColumnReader> child_reader;
- RETURN_IF_ERROR(create(file, &field->children[i], row_group, row_ranges, ctz, io_ctx,
- child_reader, max_buf_size));
+ RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx,
+ child_reader, max_buf_size, nullptr, column_ids,
+ filter_column_ids));
child_reader->set_nested_column();
- child_readers[field->children[i].name] = std::move(child_reader);
+ child_readers[field->children[0].name] = std::move(child_reader);
+ // break;
+ // }
+ // }
}
auto struct_reader = StructColumnReader::create_unique(row_ranges, ctz, io_ctx);
RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field));
+ struct_reader->_filter_column_ids = filter_column_ids;
reader.reset(struct_reader.release());
} else {
const tparquet::ColumnChunk& chunk = row_group.columns[field->physical_column_index];
auto scalar_reader =
ScalarColumnReader::create_unique(row_ranges, chunk, offset_index, ctz, io_ctx);
RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size));
+ scalar_reader->_filter_column_ids = filter_column_ids;
reader.reset(scalar_reader.release());
}
return Status::OK();
@@ -584,7 +659,8 @@ Status ScalarColumnReader::_try_load_dict_page(bool* loaded, bool* has_dict) {
Status ScalarColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
if (_converter == nullptr) {
_converter = parquet::PhysicalToLogicalConverter::get_converter(
_field_schema, _field_schema->data_type, type, _ctz, is_dict_filter);
@@ -640,7 +716,7 @@ Status ScalarColumnReader::read_column_data(
}
if (batch_size >= remaining_num_values &&
filter_map.can_filter_all(remaining_num_values, _filter_map_index)) {
- // We can skip the whole page if the remaining values is filtered by predicate columns
+ // We can skip the whole page if the remaining values are filtered by predicate columns
_filter_map_index += remaining_num_values;
_current_row_index += _chunk_reader->remaining_num_values();
RETURN_IF_ERROR(_chunk_reader->skip_page());
@@ -701,7 +777,8 @@ Status ArrayColumnReader::init(std::unique_ptr<ParquetColumnReader> element_read
Status ArrayColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
MutableColumnPtr data_column;
NullMap* null_map_ptr = nullptr;
if (doris_column->is_nullable()) {
@@ -753,7 +830,8 @@ Status MapColumnReader::init(std::unique_ptr<ParquetColumnReader> key_reader,
Status MapColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
MutableColumnPtr data_column;
NullMap* null_map_ptr = nullptr;
if (doris_column->is_nullable()) {
@@ -785,6 +863,7 @@ Status MapColumnReader::read_column_data(
size_t value_rows = 0;
bool key_eof = false;
bool value_eof = false;
+ int64_t orig_col_column_size = key_column->size();
RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->get_key_node(),
filter_map, batch_size, &key_rows, &key_eof,
@@ -794,11 +873,12 @@ Status MapColumnReader::read_column_data(
size_t loop_rows = 0;
RETURN_IF_ERROR(_value_reader->read_column_data(
value_column, value_type, root_node->get_value_node(), filter_map,
- key_rows - value_rows, &loop_rows, &value_eof, is_dict_filter));
+ key_rows - value_rows, &loop_rows, &value_eof, is_dict_filter,
+ key_column->size() - orig_col_column_size));
value_rows += loop_rows;
}
DCHECK_EQ(key_rows, value_rows);
- DCHECK_EQ(key_eof, value_eof);
+ // DCHECK_EQ(key_eof, value_eof);
*read_rows = key_rows;
*eof = key_eof;
@@ -826,7 +906,8 @@ Status StructColumnReader::init(
Status StructColumnReader::read_column_data(
ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node, FilterMap& filter_map,
- size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) {
+ size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter,
+ int64_t real_column_size) {
MutableColumnPtr data_column;
NullMap* null_map_ptr = nullptr;
if (doris_column->is_nullable()) {
@@ -850,7 +931,9 @@ Status StructColumnReader::read_column_data(
const auto* doris_struct_type = assert_cast<const DataTypeStruct*>(remove_nullable(type).get());
int64_t not_missing_column_id = -1;
+ size_t not_missing_orig_column_size = 0;
std::vector<size_t> missing_column_idxs {};
+ std::vector<size_t> skip_reading_column_idxs {};
_read_column_names.clear();
@@ -860,21 +943,48 @@ Status StructColumnReader::read_column_data(
auto& doris_name = doris_struct_type->get_element_name(i);
if (!root_node->children_column_exists(doris_name)) {
missing_column_idxs.push_back(i);
+ VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i
+ << "], doris_name: " << doris_name << " (column not exists in root node)";
continue;
}
auto file_name = root_node->children_file_column_name(doris_name);
+ // if (_child_readers.find(file_name) == _child_readers.end()) {
+ // missing_column_idxs.push_back(i);
+ // continue;
+ // }
+
+ // Check if this is a SkipReadingReader - we should skip it when choosing reference column
+ // because SkipReadingReader doesn't know the actual data size in nested context
+ bool is_skip_reader =
+ dynamic_cast<SkipReadingReader*>(_child_readers[file_name].get()) != nullptr;
+
+ if (is_skip_reader) {
+ // Store SkipReadingReader columns to fill them later based on reference column size
+ skip_reading_column_idxs.push_back(i);
+ continue;
+ }
+ // Only add non-SkipReadingReader columns to _read_column_names
+ // This ensures get_rep_level() and get_def_level() return valid levels
_read_column_names.emplace_back(file_name);
size_t field_rows = 0;
bool field_eof = false;
if (not_missing_column_id == -1) {
not_missing_column_id = i;
+ not_missing_orig_column_size = doris_field->size();
RETURN_IF_ERROR(_child_readers[file_name]->read_column_data(
doris_field, doris_type, root_node->get_children_node(doris_name), filter_map,
batch_size, &field_rows, &field_eof, is_dict_filter));
*read_rows = field_rows;
*eof = field_eof;
+ // Debug: print the first non-missing child column read info
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' first non-missing child='"
+ // << file_name << "' doris_name='" << doris_name << "' field_rows=" << field_rows
+ // << " field_eof=" << field_eof << " doris_field_size=" << doris_field->size()
+ // << " child_rep_levels=" << _child_readers[file_name]->get_rep_level().size()
+ // << " child_def_levels=" << _child_readers[file_name]->get_def_level().size()
+ // << std::endl;
/*
* Considering the issue in the `_read_nested_column` function where data may span across pages, leading
* to missing definition and repetition levels, when filling the null_map of the struct later, it is
@@ -891,6 +1001,12 @@ Status StructColumnReader::read_column_data(
doris_field, doris_type, root_node->get_children_node(doris_name),
filter_map, *read_rows - field_rows, &loop_rows, &field_eof,
is_dict_filter));
+ // Debug: print each loop iteration for non-first child columns
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' non-first child='"
+ // << file_name << "' doris_name='" << doris_name << "' loop_rows=" << loop_rows
+ // << " field_rows=" << field_rows << " target_rows=" << *read_rows
+ // << " field_eof=" << field_eof << " doris_field_size=" << doris_field->size()
+ // << std::endl;
field_rows += loop_rows;
}
DCHECK_EQ(*read_rows, field_rows);
@@ -898,10 +1014,90 @@ Status StructColumnReader::read_column_data(
}
}
+ // After scanning children, print summary of missing/skip info
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' summary: not_missing_column_id="
+ // << not_missing_column_id << " missing_columns_count=" << missing_column_idxs.size()
+ // << " skip_reading_count=" << skip_reading_column_idxs.size() << std::endl;
+
+ int64_t missing_column_sz = -1;
+
if (not_missing_column_id == -1) {
- // TODO: support read struct which columns are all missing
- return Status::Corruption("Not support read struct '{}' which columns are all missing",
- _field_schema->name);
+ // All queried columns are missing in the file (e.g., all added after schema change)
+ // We need to pick a column from _field_schema that exists in the file to get RL/DL
+ //std::cout << "[ParquetReader] All queried columns missing for struct '" << _field_schema->name
+ // << "', searching for reference column in _field_schema" << std::endl;
+
+ // Find a column from _field_schema children that exists in the file for RL/DL reference
+ std::string reference_file_column_name;
+ std::unique_ptr<ParquetColumnReader>* reference_reader = nullptr;
+
+ for (const auto& child : _field_schema->children) {
+ auto it = _child_readers.find(child.name);
+ if (it != _child_readers.end()) {
+ // Skip SkipReadingReader as they don't have valid RL/DL
+ bool is_skip_reader = dynamic_cast<SkipReadingReader*>(it->second.get()) != nullptr;
+ if (!is_skip_reader) {
+ reference_file_column_name = child.name;
+ reference_reader = &(it->second);
+ //std::cout << "[ParquetReader] Found reference column: " << reference_file_column_name
+ // << " for struct '" << _field_schema->name << "'" << std::endl;
+ break;
+ }
+ }
+ }
+
+ if (reference_reader != nullptr) {
+ // Read the reference column to get correct RL/DL information
+ // We need to find the FieldSchema for the reference column from _field_schema children
+ FieldSchema* ref_field_schema = nullptr;
+ for (auto& child : _field_schema->children) {
+ if (child.name == reference_file_column_name) {
+ ref_field_schema = &child;
+ break;
+ }
+ }
+
+ if (ref_field_schema == nullptr) {
+ return Status::InternalError(
+ "Cannot find field schema for reference column '{}' in struct '{}'",
+ reference_file_column_name, _field_schema->name);
+ }
+
+ // Create a temporary column to hold the data (we'll use its size for missing_column_sz)
+ ColumnPtr temp_column = ref_field_schema->data_type->create_column();
+ auto temp_type = ref_field_schema->data_type;
+
+ size_t field_rows = 0;
+ bool field_eof = false;
+
+ // Use root_node to get the correct child node for the reference column
+ // reference_file_column_name is the file column name, use get_children_node_by_file_column_name
+ auto ref_child_node =
+ root_node->get_children_node_by_file_column_name(reference_file_column_name);
+ not_missing_orig_column_size = temp_column->size();
+
+ RETURN_IF_ERROR((*reference_reader)
+ ->read_column_data(temp_column, temp_type, ref_child_node,
+ filter_map, batch_size, &field_rows,
+ &field_eof, is_dict_filter));
+
+ *read_rows = field_rows;
+ *eof = field_eof;
+
+ // Store this reference column name for get_rep_level/get_def_level to use
+ _read_column_names.emplace_back(reference_file_column_name);
+
+ //std::cout << "[ParquetReader] Read " << field_rows << " rows from reference column '"
+ // << reference_file_column_name << "', temp_column.size=" << temp_column->size()
+ // << " for struct '" << _field_schema->name << "'" << std::endl;
+
+ missing_column_sz = temp_column->size() - not_missing_orig_column_size;
+ } else {
+ return Status::Corruption(
+ "Cannot read struct '{}': all queried columns are missing and no reference "
+ "column found in file",
+ _field_schema->name);
+ }
}
// This missing_column_sz is not *read_rows. Because read_rows returns the number of rows.
@@ -912,8 +1108,27 @@ Status StructColumnReader::read_column_data(
// [{4,null},{5,null}]
// When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2.
// You should insert 5 records into subcolumn b instead of 2.
- auto missing_column_sz = doris_struct.get_column(not_missing_column_id).size();
- // fill missing column with null or default value
+ if (missing_column_sz == -1) {
+ missing_column_sz = doris_struct.get_column(not_missing_column_id).size() -
+ not_missing_orig_column_size;
+ }
+
+ // Fill SkipReadingReader columns with the correct amount of data based on reference column
+ // Let SkipReadingReader handle the data filling through its read_column_data method
+ for (auto idx : skip_reading_column_idxs) {
+ auto& doris_field = doris_struct.get_column_ptr(idx);
+ auto& doris_type = const_cast<DataTypePtr&>(doris_struct_type->get_element(idx));
+ auto& doris_name = const_cast<String&>(doris_struct_type->get_element_name(idx));
+ auto file_name = root_node->children_file_column_name(doris_name);
+
+ size_t field_rows = 0;
+ bool field_eof = false;
+ RETURN_IF_ERROR(_child_readers[file_name]->read_column_data(
+ doris_field, doris_type, root_node->get_children_node(doris_name), filter_map,
+ missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz));
+ }
+
+ // Fill truly missing columns (not in root_node) with null or default value
for (auto idx : missing_column_idxs) {
auto& doris_field = doris_struct.get_column_ptr(idx);
auto& doris_type = doris_struct_type->get_element(idx);
@@ -924,10 +1139,11 @@ Status StructColumnReader::read_column_data(
}
if (null_map_ptr != nullptr) {
+ //std::cout << "[ParquetReader] struct '" << _field_schema->name << "' before fill_struct_null_map: rep_levels="
+ // << this->get_rep_level().size() << " def_levels=" << this->get_def_level().size() << std::endl;
fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(),
this->get_def_level());
}
-
return Status::OK();
}
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
index f0d02b0d3e4..7b37f2e0e6c 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h
@@ -123,7 +123,8 @@ public:
virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows,
- bool* eof, bool is_dict_filter) = 0;
+ bool* eof, bool is_dict_filter,
+ int64_t real_column_size = -1) = 0;
virtual Status read_dict_values_to_column(MutableColumnPtr& doris_column, bool* has_dict) {
return Status::NotSupported("read_dict_values_to_column is not supported");
@@ -138,7 +139,9 @@ public:
const tparquet::RowGroup& row_group,
const std::vector<RowRange>& row_ranges, const cctz::time_zone* ctz,
io::IOContext* io_ctx, std::unique_ptr<ParquetColumnReader>& reader,
- size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr);
+ size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr,
+ const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
void set_nested_column() { _nested_column = true; }
virtual const std::vector<level_t>& get_rep_level() const = 0;
virtual const std::vector<level_t>& get_def_level() const = 0;
@@ -147,6 +150,8 @@ public:
virtual void reset_filter_map_index() = 0;
+ FieldSchema* get_field_schema() const { return _field_schema; }
+
protected:
void _generate_read_ranges(int64_t start_index, int64_t end_index,
std::list<RowRange>& read_ranges);
@@ -162,6 +167,7 @@ protected:
int64_t _decode_null_map_time = 0;
size_t _filter_map_index = 0;
+ std::set<uint64_t> _filter_column_ids;
};
class ScalarColumnReader : public ParquetColumnReader {
@@ -179,7 +185,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
Status read_dict_values_to_column(MutableColumnPtr& doris_column, bool* has_dict) override;
MutableColumnPtr convert_dict_column_to_string_column(const ColumnInt32* dict_column) override;
const std::vector<level_t>& get_rep_level() const override { return _rep_levels; }
@@ -226,7 +232,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
const std::vector<level_t>& get_rep_level() const override {
return _element_reader->get_rep_level();
}
@@ -255,7 +261,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
const std::vector<level_t>& get_rep_level() const override {
return _key_reader->get_rep_level();
@@ -297,7 +303,7 @@ public:
Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
- bool is_dict_filter) override;
+ bool is_dict_filter, int64_t real_column_size = -1) override;
const std::vector<level_t>& get_rep_level() const override {
if (!_read_column_names.empty()) {
@@ -346,6 +352,93 @@ private:
std::vector<std::string> _read_column_names;
//Need to use vector instead of set,see `get_rep_level()` for the reason.
};
+
+// A special reader that skips actual reading but provides empty data with correct structure
+// This is used when a column is not needed but its structure is required (e.g., for map keys)
+class SkipReadingReader : public ParquetColumnReader {
+public:
+ SkipReadingReader(const std::vector<RowRange>& row_ranges, const cctz::time_zone* ctz,
+ io::IOContext* io_ctx, FieldSchema* field_schema)
+ : ParquetColumnReader(row_ranges, ctz, io_ctx) {
+ _field_schema = field_schema; // Use inherited member from base class
+ VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: "
+ << _field_schema->name;
+ }
+
+ Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type,
+ const std::shared_ptr<TableSchemaChangeHelper::Node>& root_node,
+ FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof,
+ bool is_dict_filter, int64_t real_column_size = -1) override {
+ VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: "
+ << _field_schema->name << ", batch_size: " << batch_size;
+ DCHECK(real_column_size >= 0); // real_column_size for filtered column size.
+
+ // Simulate reading without actually reading data
+ // Fill with default/null values based on column type
+ MutableColumnPtr data_column = doris_column->assume_mutable();
+
+ if (real_column_size > 0) {
+ if (doris_column->is_nullable()) {
+ auto* nullable_column = static_cast<vectorized::ColumnNullable*>(data_column.get());
+ nullable_column->insert_many_defaults(real_column_size);
+ } else {
+ // For non-nullable columns, insert appropriate default values
+ for (size_t i = 0; i < real_column_size; ++i) {
+ data_column->insert_default();
+ }
+ }
+ }
+
+ *read_rows = batch_size; // Indicate we "read" batch_size rows
+ *eof = false; // We can always provide more empty data
+
+ VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size
+ << " default values for field: " << _field_schema->name;
+
+ return Status::OK();
+ }
+
+ static std::unique_ptr<SkipReadingReader> create_unique(const std::vector<RowRange>& row_ranges,
+ cctz::time_zone* ctz,
+ io::IOContext* io_ctx,
+ FieldSchema* field_schema) {
+ return std::make_unique<SkipReadingReader>(row_ranges, ctz, io_ctx, field_schema);
+ }
+
+ // These methods should not be called for SkipReadingReader
+ // If they are called, it indicates a logic error in the code
+ const std::vector<level_t>& get_rep_level() const override {
+ LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: "
+ << _field_schema->name
+ << ". This indicates the SkipReadingReader was incorrectly used as a reference "
+ "column.";
+ // static std::vector<level_t> empty_levels;
+ // return empty_levels;
+ __builtin_unreachable();
+ }
+
+ const std::vector<level_t>& get_def_level() const override {
+ LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: "
+ << _field_schema->name
+ << ". This indicates the SkipReadingReader was incorrectly used as a reference "
+ "column.";
+ // static std::vector<level_t> empty_levels;
+ // return empty_levels;
+ __builtin_unreachable();
+ }
+
+ // Implement required pure virtual methods from base class
+ Statistics statistics() override {
+ return Statistics(); // Return empty statistics
+ }
+
+ void close() override {
+ // Nothing to close for skip reading
+ }
+
+ void reset_filter_map_index() override { _filter_map_index = 0; }
+};
+
#include "common/compile_check_end.h"
}; // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
index 7d1f9d27430..19ddb8d1b99 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp
@@ -25,6 +25,7 @@
#include <algorithm>
#include <boost/iterator/iterator_facade.hpp>
+#include <iostream>
#include <ostream>
#include "common/config.h"
@@ -82,7 +83,9 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
const int32_t row_group_id, const tparquet::RowGroup& row_group,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
const PositionDeleteContext& position_delete_ctx,
- const LazyReadContext& lazy_read_ctx, RuntimeState* state)
+ const LazyReadContext& lazy_read_ctx, RuntimeState* state,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids)
: _file_reader(file_reader),
_read_table_columns(read_columns),
_row_group_id(row_group_id),
@@ -93,7 +96,15 @@ RowGroupReader::RowGroupReader(io::FileReaderSPtr file_reader,
_position_delete_ctx(position_delete_ctx),
_lazy_read_ctx(lazy_read_ctx),
_state(state),
- _obj_pool(new ObjectPool()) {}
+ _obj_pool(new ObjectPool()),
+ _column_ids(column_ids),
+ _filter_column_ids(filter_column_ids) {
+ //std::cout << "[RowGroupReader] _column_ids: ";
+ //for (const auto& col_id : _column_ids) {
+ // std::cout << col_id << " ";
+ //}
+ //std::cout << std::endl;
+}
RowGroupReader::~RowGroupReader() {
_column_readers.clear();
@@ -130,9 +141,9 @@ Status RowGroupReader::init(
const tparquet::OffsetIndex* offset_index =
col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index]
: nullptr;
- RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field, _row_group_meta,
- _read_ranges, _ctz, _io_ctx, reader,
- max_buf_size, offset_index));
+ RETURN_IF_ERROR(ParquetColumnReader::create(
+ _file_reader, field, _row_group_meta, _read_ranges, _ctz, _io_ctx, reader,
+ max_buf_size, offset_index, _column_ids, _filter_column_ids));
if (reader == nullptr) {
VLOG_DEBUG << "Init row group(" << _row_group_id << ") reader failed";
return Status::Corruption("Init row group reader failed");
@@ -320,8 +331,8 @@ Status RowGroupReader::next_batch(Block* block, size_t batch_size, size_t* read_
return _do_lazy_read(block, batch_size, read_rows, batch_eof);
} else {
FilterMap filter_map;
- RETURN_IF_ERROR(_read_column_data(block, _lazy_read_ctx.all_read_columns, batch_size,
- read_rows, batch_eof, filter_map));
+ RETURN_IF_ERROR((_read_column_data(block, _lazy_read_ctx.all_read_columns, batch_size,
+ read_rows, batch_eof, filter_map)));
RETURN_IF_ERROR(
_fill_partition_columns(block, *read_rows, _lazy_read_ctx.partition_columns));
RETURN_IF_ERROR(_fill_missing_columns(block, *read_rows, _lazy_read_ctx.missing_columns));
@@ -427,9 +438,19 @@ Status RowGroupReader::_read_column_data(Block* block,
RETURN_IF_ERROR(_column_readers[read_col_name]->read_column_data(
column_ptr, column_type, _table_info_node_ptr->get_children_node(read_col_name),
filter_map, batch_size - col_read_rows, &loop_rows, &col_eof, is_dict_filter));
+ // Debug: print loop_rows for this iteration
+ //std::cout << "[RowGroupReader] column '" << read_col_name << "' loop_rows=" << loop_rows
+ // << " col_read_rows_so_far=" << col_read_rows << std::endl;
col_read_rows += loop_rows;
}
+ // Debug: print rows read for this column
+ //std::cout << "[RowGroupReader] column '" << read_col_name << "' read_rows=" << col_read_rows
+ // << std::endl;
if (batch_read_rows > 0 && batch_read_rows != col_read_rows) {
+ LOG(WARNING) << "[RowGroupReader] Mismatched read rows among parquet columns. "
+ "previous_batch_read_rows="
+ << batch_read_rows << ", current_column='" << read_col_name
+ << "', current_col_read_rows=" << col_read_rows;
return Status::Corruption("Can't read the same number of rows among parquet columns");
}
batch_read_rows = col_read_rows;
@@ -720,6 +741,10 @@ Status RowGroupReader::_fill_missing_columns(
block->erase(result_column_id);
}
}
+
+ // auto mutable_column = block->get_by_name(kv.first).column->assume_mutable();
+ // auto* nullable_column = assert_cast<vectorized::ColumnNullable*>(mutable_column.get());
+ // nullable_column->insert_many_defaults(rows);
}
return Status::OK();
}
@@ -1110,6 +1135,7 @@ ParquetColumnReader::Statistics RowGroupReader::statistics() {
}
return st;
}
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
index 86b7cdf3f0e..5853cd6827d 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
@@ -150,7 +150,9 @@ public:
const int32_t row_group_id, const tparquet::RowGroup& row_group,
const cctz::time_zone* ctz, io::IOContext* io_ctx,
const PositionDeleteContext& position_delete_ctx,
- const LazyReadContext& lazy_read_ctx, RuntimeState* state);
+ const LazyReadContext& lazy_read_ctx, RuntimeState* state,
+ const std::set<uint64_t>& column_ids,
+ const std::set<uint64_t>& filter_column_ids);
~RowGroupReader();
Status init(const FieldDescriptor& schema, std::vector<RowRange>& row_ranges,
@@ -188,9 +190,11 @@ private:
void _merge_read_ranges(std::vector<RowRange>& row_ranges);
Status _read_empty_batch(size_t batch_size, size_t* read_rows, bool* batch_eof,
bool* modify_row_ids);
+
Status _read_column_data(Block* block, const std::vector<std::string>& columns,
size_t batch_size, size_t* read_rows, bool* batch_eof,
FilterMap& filter_map);
+
Status _do_lazy_read(Block* block, size_t batch_size, size_t* read_rows, bool* batch_eof);
Status _rebuild_filter_map(FilterMap& filter_map,
DorisUniqueBufferPtr<uint8_t>& filter_map_data,
@@ -249,6 +253,8 @@ private:
std::vector<std::pair<std::string, int>> _dict_filter_cols;
RuntimeState* _state = nullptr;
std::shared_ptr<ObjectPool> _obj_pool;
+ const std::set<uint64_t>& _column_ids;
+ const std::set<uint64_t>& _filter_column_ids;
bool _is_row_group_filtered = false;
RowGroupIndex _current_row_group_idx {0, 0, 0};
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 5accadf77f7..e47a0eddaac 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -319,7 +319,8 @@ Status ParquetReader::init_reader(
const std::unordered_map<std::string, int>* colname_to_slot_id,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
- std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr, bool filter_groups) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr, bool filter_groups,
+ const std::set<uint64_t>& column_ids, const std::set<uint64_t>& filter_column_ids) {
_tuple_descriptor = tuple_descriptor;
_row_descriptor = row_descriptor;
_colname_to_slot_id = colname_to_slot_id;
@@ -328,6 +329,8 @@ Status ParquetReader::init_reader(
_colname_to_value_range = colname_to_value_range;
_table_info_node_ptr = table_info_node_ptr;
_filter_groups = filter_groups;
+ _column_ids = column_ids;
+ _filter_column_ids = filter_column_ids;
RETURN_IF_ERROR(_open_file());
_t_metadata = &(_file_metadata->to_thrift());
@@ -706,12 +709,12 @@ Status ParquetReader::_next_row_group_reader() {
_profile, _file_reader, io_ranges, merged_read_slice_size)
: _file_reader;
}
- _current_group_reader.reset(
- new RowGroupReader(_io_ctx ? std::make_shared<io::TracingFileReader>(
- group_file_reader, _io_ctx->file_reader_stats)
- : group_file_reader,
- _read_table_columns, row_group_index.row_group_id, row_group, _ctz,
- _io_ctx, position_delete_ctx, _lazy_read_ctx, _state));
+ _current_group_reader.reset(new RowGroupReader(
+ _io_ctx ? std::make_shared<io::TracingFileReader>(group_file_reader,
+ _io_ctx->file_reader_stats)
+ : group_file_reader,
+ _read_table_columns, row_group_index.row_group_id, row_group, _ctz, _io_ctx,
+ position_delete_ctx, _lazy_read_ctx, _state, _column_ids, _filter_column_ids));
_row_group_eof = false;
_current_group_reader->set_current_row_group_idx(row_group_index);
@@ -796,8 +799,8 @@ std::vector<io::PrefetchRange> ParquetReader::_generate_random_access_ranges(
if (field->data_type->get_primitive_type() == TYPE_ARRAY) {
scalar_range(&field->children[0], row_group);
} else if (field->data_type->get_primitive_type() == TYPE_MAP) {
- scalar_range(&field->children[0].children[0], row_group);
- scalar_range(&field->children[0].children[1], row_group);
+ scalar_range(&field->children[0], row_group);
+ scalar_range(&field->children[1], row_group);
} else if (field->data_type->get_primitive_type() == TYPE_STRUCT) {
for (int i = 0; i < field->children.size(); ++i) {
scalar_range(&field->children[i], row_group);
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h
index fecc433dba6..689c45a37d8 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.h
@@ -123,7 +123,8 @@ public:
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts,
std::shared_ptr<TableSchemaChangeHelper::Node> table_info_node_ptr =
TableSchemaChangeHelper::ConstNode::get_instance(),
- bool filter_groups = true);
+ bool filter_groups = true, const std::set<uint64_t>& column_ids = {},
+ const std::set<uint64_t>& filter_column_ids = {});
Status get_next_block(Block* block, size_t* read_rows, bool* eof) override;
@@ -316,6 +317,8 @@ private:
-1};
bool _filter_groups;
+ std::set<uint64_t> _column_ids;
+ std::set<uint64_t> _filter_column_ids;
// for page index filter. slot id => expr
std::map<int, std::vector<std::unique_ptr<ColumnPredicate>>> _push_down_simple_predicates;
diff --git a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
new file mode 100644
index 00000000000..6e9fa381fc7
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive/hive_orc_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/logging.h"
+#include "orc/Type.hh"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void HiveOrcNestedColumnUtils::extract_nested_column_ids(
+ const orc::Type& type, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>>
+ child_paths_by_table_col_name;
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_table_col_name = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_table_col_name[first_table_col_name].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (type.getKind() == orc::TypeKind::MAP) {
+ auto wildcard_it = child_paths_by_table_col_name.find("*");
+ if (wildcard_it != child_paths_by_table_col_name.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_table_col_name["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_table_col_name["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_table_col_name.erase(wildcard_it);
+ }
+ }
+
+ for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) {
+ const orc::Type* child = type.getSubtype(i);
+
+ std::string child_field_name;
+ switch (type.getKind()) {
+ case orc::TypeKind::STRUCT:
+ child_field_name = to_lower(type.getFieldName(i));
+ break;
+ case orc::TypeKind::LIST:
+ child_field_name = "*";
+ break;
+ case orc::TypeKind::MAP:
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_name = "KEYS";
+ } else if (i == 1) {
+ child_field_name = "VALUES";
+ }
+ break;
+ default:
+ child_field_name = "";
+ break;
+ }
+
+ if (child_field_name.empty()) {
+ continue;
+ }
+
+ const auto child_paths_it = child_paths_by_table_col_name.find(child_field_name);
+
+ if (child_paths_it != child_paths_by_table_col_name.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ uint64_t start_id = child->getColumnId();
+ uint64_t max_column_id = child->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(*child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ };
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(type.getColumnId());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h
new file mode 100644
index 00000000000..1e1968c09b9
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace orc {
+class Type;
+} // namespace orc
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class HiveOrcNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const orc::Type& type,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp
new file mode 100644
index 00000000000..a0052ecfbfc
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp
@@ -0,0 +1,172 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive/hive_parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/parquet/schema_desc.h"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void HiveParquetNestedColumnUtils::extract_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>>
+ child_paths_by_table_col_name;
+
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_table_col_name = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_table_col_name[first_table_col_name].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP) {
+ auto wildcard_it = child_paths_by_table_col_name.find("*");
+ if (wildcard_it != child_paths_by_table_col_name.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_table_col_name["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_table_col_name["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_table_col_name.erase(wildcard_it);
+ }
+ }
+
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < field_schema.children.size(); ++i) {
+ const auto& child = field_schema.children[i];
+ std::string child_field_name;
+
+ bool is_list = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY;
+ bool is_map = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP;
+
+ if (is_list) {
+ child_field_name = "*";
+ } else if (is_map) {
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_name = "KEYS";
+ } else if (i == 1) {
+ child_field_name = "VALUES";
+ }
+
+ // Special handling for Parquet MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access = child_paths_by_table_col_name.find("KEYS") !=
+ child_paths_by_table_col_name.end();
+ bool has_values_access = child_paths_by_table_col_name.find("VALUES") !=
+ child_paths_by_table_col_name.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for RL/DL
+ if (!has_keys_access && has_values_access) {
+ // For map_values() queries, we need key's structure for correct RL/DL parsing.
+ // If key is a nested type (e.g., STRUCT), RL/DL info is stored at leaf columns.
+ // Add all column IDs from key's start to max (all leaves + intermediate nodes).
+ uint64_t key_start_id = child.get_column_id();
+ uint64_t key_max_id = child.get_max_column_id();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
+
+ } else {
+ child_field_name = child.lower_case_name;
+ }
+
+ if (child_field_name.empty()) {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_table_col_name.find(child_field_name);
+ if (child_paths_it != child_paths_by_table_col_name.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ // Add all column IDs from current child node to max_column_id
+ // This efficiently handles all nested/complex cases in one loop
+ uint64_t start_id = child.get_column_id();
+ uint64_t max_column_id = child.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(field_schema.get_column_id());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h
new file mode 100644
index 00000000000..6b57a0a3a38
--- /dev/null
+++ b/be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+struct FieldSchema;
+
+class HiveParquetNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/hive_reader.cpp b/be/src/vec/exec/format/table/hive_reader.cpp
index a454118b244..96ca2dff2d0 100644
--- a/be/src/vec/exec/format/table/hive_reader.cpp
+++ b/be/src/vec/exec/format/table/hive_reader.cpp
@@ -21,6 +21,8 @@
#include "common/status.h"
#include "runtime/runtime_state.h"
+#include "vec/exec/format/table/hive/hive_orc_nested_column_utils.h"
+#include "vec/exec/format/table/hive/hive_parquet_nested_column_utils.h"
namespace doris::vectorized {
#include "common/compile_check_begin.h"
@@ -77,10 +79,223 @@ Status HiveOrcReader::init_reader(
}
}
+ auto column_id_result = ColumnIdResult();
+ if (_state->query_options().hive_orc_use_column_names && !is_hive_col_name) {
+ column_id_result = _create_column_ids(orc_type_ptr, tuple_descriptor);
+ } else {
+ column_id_result =
+ _create_column_ids_by_top_level_col_index(orc_type_ptr, tuple_descriptor);
+ }
+
+ // const auto& file_col_names = column_id_result.column_names;
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
return orc_reader->init_reader(&read_table_col_names, table_col_name_to_value_range, conjuncts,
false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts,
- table_info_node_ptr);
+ table_info_node_ptr, column_ids, filter_column_ids);
+}
+
+ColumnIdResult HiveOrcReader::_create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column name (lower-cased) -> orc::Type*
+ std::unordered_map<std::string, const orc::Type*> table_col_name_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ std::string table_col_name = to_lower(orc_type->getFieldName(i));
+ table_col_name_to_orc_type_map[table_col_name] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(orc_field->getColumnId());
+ HiveOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_name_to_orc_type_map.find(slot->col_name());
+ if (it == table_col_name_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(orc_field, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
+}
+
+ColumnIdResult HiveOrcReader::_create_column_ids_by_top_level_col_index(
+ const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column index -> orc::Type*
+ std::unordered_map<uint64_t, const orc::Type*> table_col_pos_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ table_col_pos_to_orc_type_map[i] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(orc_field->getColumnId());
+ HiveOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = table_col_pos_to_orc_type_map.find(slot->col_pos());
+ if (it == table_col_pos_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by pos
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // complex types
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(orc_field, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status HiveParquetReader::init_reader(
@@ -145,10 +360,238 @@ Status HiveParquetReader::init_reader(
}
}
- return parquet_reader->init_reader(read_table_col_names, table_col_name_to_value_range,
- conjuncts, tuple_descriptor, row_descriptor,
- colname_to_slot_id, not_single_slot_filter_conjuncts,
- slot_id_to_filter_conjuncts, table_info_node_ptr);
+ auto column_id_result = ColumnIdResult();
+ if (_state->query_options().hive_parquet_use_column_names) {
+ column_id_result = _create_column_ids(field_desc, tuple_descriptor);
+ } else {
+ column_id_result = _create_column_ids_by_top_level_col_index(field_desc, tuple_descriptor);
+ }
+
+ const auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
+ RETURN_IF_ERROR(init_row_filters());
+
+ return parquet_reader->init_reader(
+ read_table_col_names, table_col_name_to_value_range, conjuncts, tuple_descriptor,
+ row_descriptor, colname_to_slot_id, not_single_slot_filter_conjuncts,
+ slot_id_to_filter_conjuncts, table_info_node_ptr, true, column_ids, filter_column_ids);
+}
+
+ColumnIdResult HiveParquetReader::_create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<std::string, const FieldSchema*> table_col_name_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ table_col_name_to_field_schema_map[field_schema->lower_case_name] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(parquet_field->get_column_id());
+ HiveParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+
+ // Find the field schema for this slot (may not exist for partition columns, etc.)
+ auto it = table_col_name_to_field_schema_map.find(slot->col_name());
+ if (it == table_col_name_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(field_schema, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
+}
+
+ColumnIdResult HiveParquetReader::_create_column_ids_by_top_level_col_index(
+ const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<uint64_t, const FieldSchema*> table_col_pos_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ table_col_pos_to_field_schema_map[i] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(parquet_field->get_column_id());
+ HiveParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+
+ // Find the field schema for this slot (may not exist for partition columns, etc.)
+ auto it = table_col_pos_to_field_schema_map.find(slot->col_pos());
+ if (it == table_col_pos_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by position
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(field_schema, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/format/table/hive_reader.h b/be/src/vec/exec/format/table/hive_reader.h
index 6acd344e417..b9592c43243 100644
--- a/be/src/vec/exec/format/table/hive_reader.h
+++ b/be/src/vec/exec/format/table/hive_reader.h
@@ -66,6 +66,13 @@ public:
const RowDescriptor* row_descriptor,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts);
+
+private:
+ static ColumnIdResult _create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor);
+
+ static ColumnIdResult _create_column_ids_by_top_level_col_index(
+ const orc::Type* orc_type, const TupleDescriptor* tuple_descriptor);
};
class HiveParquetReader final : public HiveReader {
@@ -88,6 +95,13 @@ public:
const std::unordered_map<std::string, int>* colname_to_slot_id,
const VExprContextSPtrs* not_single_slot_filter_conjuncts,
const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts);
+
+private:
+ static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor);
+
+ static ColumnIdResult _create_column_ids_by_top_level_col_index(
+ const FieldDescriptor* field_desc, const TupleDescriptor* tuple_descriptor);
};
#include "common/compile_check_end.h"
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
new file mode 100644
index 00000000000..5dbbbfb2af2
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "orc/Type.hh"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void IcebergOrcNestedColumnUtils::extract_nested_column_ids(
+ const orc::Type& type, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>> child_paths_by_field_id;
+
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_field_id = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_field_id[first_field_id].push_back(std::move(remaining));
+ }
+ }
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (type.getKind() == orc::TypeKind::MAP) {
+ auto wildcard_it = child_paths_by_field_id.find("*");
+ if (wildcard_it != child_paths_by_field_id.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_field_id["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_field_id["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_field_id.erase(wildcard_it);
+ }
+ }
+
+ bool has_child_columns = false;
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) {
+ const orc::Type* child = type.getSubtype(i);
+
+ std::string child_field_id;
+ switch (type.getKind()) {
+ case orc::TypeKind::STRUCT:
+ if (!child->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) {
+ continue;
+ }
+ child_field_id = child->getAttributeValue(ICEBERG_ORC_ATTRIBUTE);
+ break;
+ case orc::TypeKind::LIST:
+ child_field_id = "*";
+ break;
+ case orc::TypeKind::MAP:
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_id = "KEYS";
+ } else if (i == 1) {
+ child_field_id = "VALUES";
+ }
+ break;
+ default:
+ child_field_id = "";
+ break;
+ }
+
+ if (child_field_id.empty() || child_field_id == "-1") {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_field_id.find(child_field_id);
+ if (child_paths_it != child_paths_by_field_id.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ uint64_t start_id = child->getColumnId();
+ uint64_t max_column_id = child->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(*child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(type.getColumnId());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h
new file mode 100644
index 00000000000..576ec598970
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h
@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace orc {
+class Type;
+} // namespace orc
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class IcebergOrcNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const orc::Type& type,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+
+private:
+ static constexpr const char* ICEBERG_ORC_ATTRIBUTE = "iceberg.id";
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
new file mode 100644
index 00000000000..cb2b2e6b8c7
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h"
+
+#include <algorithm>
+#include <iostream>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/parquet/schema_desc.h"
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+void IcebergParquetNestedColumnUtils::extract_nested_column_ids(
+ const FieldSchema& field_schema, const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids) {
+ // Group paths by first field_id
+ std::unordered_map<std::string, std::vector<std::vector<std::string>>> child_paths_by_field_id;
+
+ for (const auto& path : paths) {
+ if (!path.empty()) {
+ std::string first_field_id = path[0];
+ std::vector<std::string> remaining;
+ if (path.size() > 1) {
+ remaining.assign(path.begin() + 1, path.end());
+ }
+ child_paths_by_field_id[first_field_id].push_back(std::move(remaining));
+ }
+ }
+
+ // Track whether any child column was added to determine if parent should be included
+ bool has_child_columns = false;
+
+ // For MAP type, normalize wildcard "*" to explicit KEYS/VALUES access
+ // Wildcard in MAP context means accessing both map keys and values
+ // Normalization logic:
+ // path: ["map_col", "*"] → ["map_col", "VALUES"] + ["map_col", "KEYS"]
+ // path: ["map_col", "*", "field"] → ["map_col", "VALUES", "field"] + ["map_col", "KEYS"]
+ // KEYS are always needed for correct RL/DL computation when accessing MAP via wildcard
+ if (field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP) {
+ auto wildcard_it = child_paths_by_field_id.find("*");
+ if (wildcard_it != child_paths_by_field_id.end()) {
+ auto& wildcard_paths = wildcard_it->second;
+
+ // All wildcard paths go to VALUES
+ auto& values_paths = child_paths_by_field_id["VALUES"];
+ values_paths.insert(values_paths.end(), wildcard_paths.begin(), wildcard_paths.end());
+
+ // Always add KEYS for wildcard access (needed for RL/DL computation)
+ auto& keys_paths = child_paths_by_field_id["KEYS"];
+ // Add an empty path to request full KEYS
+ std::vector<std::string> empty_path;
+ keys_paths.push_back(empty_path);
+
+ // Remove wildcard entry as it's been expanded
+ child_paths_by_field_id.erase(wildcard_it);
+ }
+ }
+
+ // Efficiently traverse children
+ for (uint64_t i = 0; i < field_schema.children.size(); ++i) {
+ const auto& child = field_schema.children[i];
+
+ std::string child_field_id;
+
+ bool is_list = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY;
+ bool is_map = field_schema.data_type->get_primitive_type() == PrimitiveType::TYPE_MAP;
+
+ if (is_list) {
+ child_field_id = "*";
+ } else if (is_map) {
+ // After wildcard normalization above, all MAP accesses are explicit KEYS/VALUES
+ // Simply assign the appropriate field name based on which child we're processing
+ if (i == 0) {
+ child_field_id = "KEYS";
+ } else if (i == 1) {
+ child_field_id = "VALUES";
+ }
+
+ // Special handling for Parquet MAP structure:
+ // When accessing only VALUES, we still need KEY structure for levels
+ // Check if we're at key child (i==0) and only VALUES is requested (no KEYS)
+ if (i == 0) {
+ bool has_keys_access =
+ child_paths_by_field_id.find("KEYS") != child_paths_by_field_id.end();
+ bool has_values_access =
+ child_paths_by_field_id.find("VALUES") != child_paths_by_field_id.end();
+
+ // If only VALUES is accessed (not KEYS), still include key structure for RL/DL
+ if (!has_keys_access && has_values_access) {
+ // For map_values() queries, we need key's structure for correct RL/DL parsing.
+ // If key is a nested type (e.g., STRUCT), RL/DL info is stored at leaf columns.
+ // Add all column IDs from key's start to max (all leaves + intermediate nodes).
+ uint64_t key_start_id = child.get_column_id();
+ uint64_t key_max_id = child.get_max_column_id();
+ for (uint64_t id = key_start_id; id <= key_max_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ continue; // Skip further processing of key child
+ }
+ }
+
+ } else {
+ child_field_id = std::to_string(child.field_id);
+ }
+
+ if (child_field_id.empty() || child_field_id == "-1") {
+ continue;
+ }
+
+ auto child_paths_it = child_paths_by_field_id.find(child_field_id);
+ if (child_paths_it != child_paths_by_field_id.end()) {
+ const auto& child_paths = child_paths_it->second;
+
+ // Check if any child path is empty (meaning full child needed)
+ bool needs_full_child =
+ std::any_of(child_paths.begin(), child_paths.end(),
+ [](const std::vector<std::string>& path) { return path.empty(); });
+
+ if (needs_full_child) {
+ // Add all column IDs from current child node to max_column_id
+ // This efficiently handles all nested/complex cases in one loop
+ uint64_t start_id = child.get_column_id();
+ uint64_t max_column_id = child.get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ column_ids.insert(id);
+ }
+ has_child_columns = true;
+ } else {
+ // Store current size to check if recursive call added any columns
+ size_t before_size = column_ids.size();
+
+ // Recursively extract from child
+ extract_nested_column_ids(child, child_paths, column_ids);
+
+ // Check if recursive call added any columns
+ if (column_ids.size() > before_size) {
+ has_child_columns = true;
+ }
+ }
+ }
+ }
+
+ // If any child columns were added, also add the parent column ID
+ // This ensures parent struct/container nodes are included when their children are needed
+ if (has_child_columns) {
+ // Set automatically handles deduplication, so no need to check if it already exists
+ column_ids.insert(field_schema.get_column_id());
+ }
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h
new file mode 100644
index 00000000000..58237d8182b
--- /dev/null
+++ b/be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "vec/exec/format/table/table_format_reader.h"
+
+namespace doris::vectorized {
+#include "common/compile_check_begin.h"
+
+class FieldDescriptor;
+struct FieldSchema;
+
+class IcebergParquetNestedColumnUtils {
+public:
+ static void extract_nested_column_ids(const FieldSchema& field_schema,
+ const std::vector<std::vector<std::string>>& paths,
+ std::set<uint64_t>& column_ids);
+};
+
+#include "common/compile_check_end.h"
+} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index db6e44c3d4b..95aa41a43e6 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -17,6 +17,7 @@
#include "iceberg_reader.h"
+#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/Metrics_types.h>
#include <gen_cpp/PlanNodes_types.h>
#include <gen_cpp/parquet_types.h>
@@ -52,6 +53,8 @@
#include "vec/exec/format/orc/vorc_reader.h"
#include "vec/exec/format/parquet/schema_desc.h"
#include "vec/exec/format/parquet/vparquet_column_chunk_reader.h"
+#include "vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h"
+#include "vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h"
#include "vec/exec/format/table/table_format_reader.h"
namespace cctz {
@@ -445,11 +448,134 @@ Status IcebergParquetReader::init_reader(
}
_all_required_col_names = file_col_names;
+
+ auto column_id_result = _create_column_ids(field_desc, tuple_descriptor);
+ // const auto& file_col_names = column_id_result.column_names;
+ auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
RETURN_IF_ERROR(init_row_filters());
- return parquet_reader->init_reader(_all_required_col_names, colname_to_value_range, conjuncts,
- tuple_descriptor, row_descriptor, colname_to_slot_id,
- not_single_slot_filter_conjuncts,
- slot_id_to_filter_conjuncts, table_info_node_ptr);
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ std::string col_name = field_schema->name;
+ if (std::find(_expand_col_names.begin(), _expand_col_names.end(), col_name) !=
+ _expand_col_names.end()) {
+ column_ids.insert(field_schema->get_column_id());
+ }
+ }
+ return parquet_reader->init_reader(
+ _all_required_col_names, colname_to_value_range, conjuncts, tuple_descriptor,
+ row_descriptor, colname_to_slot_id, not_single_slot_filter_conjuncts,
+ slot_id_to_filter_conjuncts, table_info_node_ptr, true, column_ids, filter_column_ids);
+}
+
+ColumnIdResult IcebergParquetReader::_create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor) {
+ if (!field_desc) {
+ return ColumnIdResult();
+ }
+
+ // First, assign column IDs to the field descriptor
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ // Create a mapping from iceberg field_id to FieldSchema for quick lookup
+ std::unordered_map<int, const FieldSchema*> iceberg_id_to_field_schema_map;
+
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) continue;
+
+ int iceberg_id = field_schema->field_id;
+ iceberg_id_to_field_schema_map[iceberg_id] = field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level parquet field
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!parquet_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = parquet_field->get_column_id();
+ uint64_t max_column_id = parquet_field->get_max_column_id();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(parquet_field->get_column_id());
+ IcebergParquetNestedColumnUtils::extract_nested_column_ids(*parquet_field, paths,
+ out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ // if (slot->col_name().starts_with(BeConsts::GLOBAL_ROWID_COL)) {
+ // continue;
+ // }
+ auto it = iceberg_id_to_field_schema_map.find(slot->col_unique_id());
+ if (it == iceberg_id_to_field_schema_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ auto field_schema = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(field_schema->column_id);
+
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(field_schema->column_id);
+ }
+ continue;
+ }
+
+ // complex types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(field_schema, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(field_schema, predicate_access_paths, filter_column_ids);
+ }
+ }
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status IcebergParquetReader ::_read_position_delete_file(const TFileRangeDesc* delete_range,
@@ -527,11 +653,128 @@ Status IcebergOrcReader::init_reader(
}
}
+ auto column_id_result = _create_column_ids(orc_type_ptr, tuple_descriptor);
+ auto& column_ids = column_id_result.column_ids;
+ const auto& filter_column_ids = column_id_result.filter_column_ids;
+
RETURN_IF_ERROR(init_row_filters());
+ for (uint64_t i = 0; i < orc_type_ptr->getSubtypeCount(); ++i) {
+ const orc::Type* sub_type = orc_type_ptr->getSubtype(i);
+ std::string col_name = orc_type_ptr->getFieldName(i);
+ if (std::find(_expand_col_names.begin(), _expand_col_names.end(), col_name) !=
+ _expand_col_names.end()) {
+ column_ids.insert(sub_type->getColumnId());
+ }
+ }
return orc_reader->init_reader(&_all_required_col_names, colname_to_value_range, conjuncts,
false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts,
- table_info_node_ptr);
+ table_info_node_ptr, column_ids, filter_column_ids);
+}
+
+ColumnIdResult IcebergOrcReader::_create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor) {
+ std::shared_ptr<TableSchemaChangeHelper::Node> schema_node = nullptr;
+
+ if (!orc_type) {
+ return ColumnIdResult();
+ }
+
+ // map top-level table column name (lower-cased) -> orc::Type*
+ std::unordered_map<int, const orc::Type*> iceberg_id_to_orc_type_map;
+ for (uint64_t i = 0; i < orc_type->getSubtypeCount(); ++i) {
+ auto orc_sub_type = orc_type->getSubtype(i);
+ if (!orc_sub_type) continue;
+
+ if (!orc_sub_type->hasAttributeKey(ICEBERG_ORC_ATTRIBUTE)) {
+ continue;
+ }
+ int iceberg_id = std::stoi(orc_sub_type->getAttributeValue(ICEBERG_ORC_ATTRIBUTE));
+ iceberg_id_to_orc_type_map[iceberg_id] = orc_sub_type;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ // helper to process name access paths for a given top-level orc field
+ auto process_access_paths = [](const orc::Type* orc_field,
+ const std::vector<TColumnAccessPath>& access_paths,
+ std::set<uint64_t>& out_ids) {
+ if (!orc_field) return;
+ if (access_paths.empty()) return;
+
+ std::vector<std::vector<std::string>> paths;
+ bool has_top_level_only = false;
+ for (const auto& access_path : access_paths) {
+ std::vector<std::string> path;
+ if (access_path.type == TAccessPathType::DATA) {
+ path = access_path.data_access_path.path;
+ } else if (access_path.type == TAccessPathType::META) {
+ path = access_path.meta_access_path.path;
+ } else {
+ continue;
+ }
+ DCHECK(path.size() >= 1);
+ std::vector<std::string> remaining_path;
+ if (path.size() > 1) {
+ remaining_path.assign(path.begin() + 1, path.end());
+ } else {
+ // only top-level column name => means whole field
+ remaining_path = std::vector<std::string>();
+ }
+ if (remaining_path.empty()) {
+ has_top_level_only = true;
+ }
+ paths.push_back(std::move(remaining_path));
+ }
+
+ if (has_top_level_only) {
+ uint64_t start_id = orc_field->getColumnId();
+ uint64_t max_column_id = orc_field->getMaximumColumnId();
+ for (uint64_t id = start_id; id <= max_column_id; ++id) {
+ out_ids.insert(id);
+ }
+ } else if (!paths.empty()) {
+ out_ids.insert(orc_field->getColumnId());
+ IcebergOrcNestedColumnUtils::extract_nested_column_ids(*orc_field, paths, out_ids);
+ }
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it = iceberg_id_to_orc_type_map.find(slot->col_unique_id());
+ if (it == iceberg_id_to_orc_type_map.end()) {
+ // Column not found in file (e.g., partition column, added column)
+ continue;
+ }
+ const orc::Type* orc_field = it->second;
+
+ const auto& all_access_paths = slot->all_access_paths();
+
+ // primitive (non-nested) types: direct mapping by name
+ if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY &&
+ slot->col_type() != TYPE_MAP)) {
+ column_ids.insert(orc_field->getColumnId());
+ if (slot->is_predicate()) {
+ filter_column_ids.insert(orc_field->getColumnId());
+ }
+ continue;
+ }
+
+ // nested types:
+
+ // collect and process all_access_paths -> column_ids
+ if (!all_access_paths.empty()) {
+ process_access_paths(orc_field, all_access_paths, column_ids);
+ }
+
+ // collect and process predicate_access_paths -> filter_column_ids
+ const auto& predicate_access_paths = slot->predicate_access_paths();
+ if (!predicate_access_paths.empty()) {
+ process_access_paths(orc_field, predicate_access_paths, filter_column_ids);
+ }
+ }
+
+ return ColumnIdResult(std::move(column_ids), std::move(filter_column_ids));
}
Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete_range,
diff --git a/be/src/vec/exec/format/table/iceberg_reader.h b/be/src/vec/exec/format/table/iceberg_reader.h
index 6b0e3fe4b3e..d123261714e 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.h
+++ b/be/src/vec/exec/format/table/iceberg_reader.h
@@ -187,6 +187,9 @@ protected:
}
private:
+ static ColumnIdResult _create_column_ids(const FieldDescriptor* field_desc,
+ const TupleDescriptor* tuple_descriptor);
+
Status _read_position_delete_file(const TFileRangeDesc* delete_range,
DeleteFile* position_delete) final;
};
@@ -226,6 +229,10 @@ protected:
_meta_cache);
}
+private:
+ static ColumnIdResult _create_column_ids(const orc::Type* orc_type,
+ const TupleDescriptor* tuple_descriptor);
+
private:
static const std::string ICEBERG_ORC_ATTRIBUTE;
};
diff --git a/be/src/vec/exec/format/table/table_format_reader.cpp b/be/src/vec/exec/format/table/table_format_reader.cpp
index b3b7e1df336..2c614777a89 100644
--- a/be/src/vec/exec/format/table/table_format_reader.cpp
+++ b/be/src/vec/exec/format/table/table_format_reader.cpp
@@ -76,8 +76,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
if (file_field.data_type->get_primitive_type() != TYPE_MAP) [[unlikely]] {
return SCHEMA_ERROR;
}
- MOCK_REMOVE(DCHECK(file_field.children.size() == 1));
- MOCK_REMOVE(DCHECK(file_field.children[0].children.size() == 2));
+ MOCK_REMOVE(DCHECK(file_field.children.size() == 2));
std::shared_ptr<TableSchemaChangeHelper::Node> key_node = nullptr;
{
@@ -85,8 +84,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
assert_cast<const DataTypeMap*>(remove_nullable(table_data_type).get())
->get_key_type());
- RETURN_IF_ERROR(
- by_parquet_name(key_type, file_field.children[0].children[0], key_node));
+ RETURN_IF_ERROR(by_parquet_name(key_type, file_field.children[0], key_node));
}
std::shared_ptr<TableSchemaChangeHelper::Node> value_node = nullptr;
@@ -95,8 +93,7 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
assert_cast<const DataTypeMap*>(remove_nullable(table_data_type).get())
->get_value_type());
- RETURN_IF_ERROR(
- by_parquet_name(value_type, file_field.children[0].children[1], value_node));
+ RETURN_IF_ERROR(by_parquet_name(value_type, file_field.children[1], value_node));
}
node = std::make_shared<TableSchemaChangeHelper::MapNode>(key_node, value_node);
break;
@@ -430,19 +427,17 @@ Status TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_field_id(
MOCK_REMOVE(DCHECK(table_schema.nestedField.map_field.key_field.field_ptr != nullptr));
MOCK_REMOVE(DCHECK(table_schema.nestedField.map_field.value_field.field_ptr != nullptr));
- MOCK_REMOVE(DCHECK(parquet_field.children.size() == 1));
- MOCK_REMOVE(DCHECK(parquet_field.children[0].children.size() == 2));
+ MOCK_REMOVE(DCHECK(parquet_field.children.size() == 2));
std::shared_ptr<TableSchemaChangeHelper::Node> key_node = nullptr;
std::shared_ptr<TableSchemaChangeHelper::Node> value_node = nullptr;
RETURN_IF_ERROR(by_parquet_field_id(*table_schema.nestedField.map_field.key_field.field_ptr,
- parquet_field.children[0].children[0], key_node,
- exist_field_id));
+ parquet_field.children[0], key_node, exist_field_id));
- RETURN_IF_ERROR(by_parquet_field_id(
- *table_schema.nestedField.map_field.value_field.field_ptr,
- parquet_field.children[0].children[1], value_node, exist_field_id));
+ RETURN_IF_ERROR(
+ by_parquet_field_id(*table_schema.nestedField.map_field.value_field.field_ptr,
+ parquet_field.children[1], value_node, exist_field_id));
node = std::make_shared<TableSchemaChangeHelper::MapNode>(key_node, value_node);
break;
diff --git a/be/src/vec/exec/format/table/table_format_reader.h b/be/src/vec/exec/format/table/table_format_reader.h
index 1f2a41af930..00442dc45f9 100644
--- a/be/src/vec/exec/format/table/table_format_reader.h
+++ b/be/src/vec/exec/format/table/table_format_reader.h
@@ -133,6 +133,13 @@ public:
throw std::logic_error("get_children_node should not be called on base TableInfoNode");
};
+ virtual std::shared_ptr<Node> get_children_node_by_file_column_name(
+ std::string file_column_name) const {
+ throw std::logic_error(
+ "get_children_node_by_file_column_name should not be called on base "
+ "TableInfoNode");
+ };
+
virtual std::string children_file_column_name(std::string table_column_name) const {
throw std::logic_error(
"children_file_column_name should not be called on base TableInfoNode");
@@ -184,6 +191,19 @@ public:
return children.at(table_column_name).node;
}
+ std::shared_ptr<Node> get_children_node_by_file_column_name(
+ std::string file_column_name) const override {
+ // Search for the child by file column name
+ for (const auto& [table_name, child] : children) {
+ if (child.exists && child.column_name == file_column_name) {
+ return child.node;
+ }
+ }
+ // Not found - throw or return nullptr
+ throw std::runtime_error("File column name '" + file_column_name +
+ "' not found in struct children");
+ }
+
std::string children_file_column_name(std::string table_column_name) const override {
DCHECK(children.contains(table_column_name));
DCHECK(children_column_exists(table_column_name));
@@ -238,6 +258,11 @@ public:
return get_instance();
};
+ std::shared_ptr<Node> get_children_node_by_file_column_name(
+ std::string file_column_name) const override {
+ return get_instance();
+ };
+
std::string children_file_column_name(std::string table_column_name) const override {
return table_column_name;
}
@@ -388,5 +413,16 @@ public:
};
};
+struct ColumnIdResult {
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ ColumnIdResult() = default; // Add default constructor
+
+ ColumnIdResult(std::set<uint64_t> column_ids_, std::set<uint64_t> filter_column_ids_)
+ : column_ids(std::move(column_ids_)),
+ filter_column_ids(std::move(filter_column_ids_)) {}
+};
+
#include "common/compile_check_end.h"
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp
index c0b440fa6f3..1ca195c319e 100644
--- a/be/src/vec/exec/scan/file_scanner.cpp
+++ b/be/src/vec/exec/scan/file_scanner.cpp
@@ -335,7 +335,7 @@ Status FileScanner::_process_runtime_filters_partition_prune(bool& can_filter_al
return Status::OK();
}
-Status FileScanner::_process_conjuncts_for_dict_filter() {
+Status FileScanner::_process_conjuncts() {
_slot_id_to_filter_conjuncts.clear();
_not_single_slot_filter_conjuncts.clear();
for (auto& conjunct : _push_down_conjuncts) {
@@ -373,7 +373,7 @@ Status FileScanner::_process_late_arrival_conjuncts() {
for (size_t i = 0; i != _conjuncts.size(); ++i) {
RETURN_IF_ERROR(_conjuncts[i]->clone(_state, _push_down_conjuncts[i]));
}
- RETURN_IF_ERROR(_process_conjuncts_for_dict_filter());
+ RETURN_IF_ERROR(_process_conjuncts());
_discard_conjuncts();
}
if (_applied_rf_num == _total_rf_num) {
@@ -386,6 +386,8 @@ void FileScanner::_get_slot_ids(VExpr* expr, std::vector<int>* slot_ids) {
for (auto& child_expr : expr->children()) {
if (child_expr->is_slot_ref()) {
VSlotRef* slot_ref = reinterpret_cast<VSlotRef*>(child_expr.get());
+ SlotDescriptor* slot_desc = _state->desc_tbl().get_slot_descriptor(slot_ref->slot_id());
+ slot_desc->set_is_predicate(true);
slot_ids->emplace_back(slot_ref->slot_id());
} else {
_get_slot_ids(child_expr.get(), slot_ids);
diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h
index 1a44d06c77b..8c4682ca89f 100644
--- a/be/src/vec/exec/scan/file_scanner.h
+++ b/be/src/vec/exec/scan/file_scanner.h
@@ -252,7 +252,7 @@ private:
void _init_runtime_filter_partition_prune_ctxs();
void _init_runtime_filter_partition_prune_block();
Status _process_runtime_filters_partition_prune(bool& is_partition_pruned);
- Status _process_conjuncts_for_dict_filter();
+ Status _process_conjuncts();
Status _process_late_arrival_conjuncts();
void _get_slot_ids(VExpr* expr, std::vector<int>* slot_ids);
Status _generate_truncate_columns(bool need_to_get_parsed_schema);
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
new file mode 100644
index 00000000000..832aaef64a0
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
new file mode 100644
index 00000000000..36f589e393e
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json
new file mode 100644
index 00000000000..97208990039
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json
@@ -0,0 +1,145 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "daf54708-3539-4de4-a130-322cd4c3fdc4",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1758865887090,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zlib"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json
new file mode 100644
index 00000000000..12ec38bbdcd
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json
@@ -0,0 +1,176 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "daf54708-3539-4de4-a130-322cd4c3fdc4",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1758865954986,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zlib"
+ },
+ "current-snapshot-id" : 2197982135324349212,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 2197982135324349212,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 2197982135324349212,
+ "timestamp-ms" : 1758865954986,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0674",
+ "added-data-files" : "2",
+ "added-records" : "10",
+ "added-files-size" : "4611",
+ "changed-partition-count" : "1",
+ "total-records" : "10",
+ "total-files-size" : "4611",
+ "total-data-files" : "2",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1758865954986,
+ "snapshot-id" : 2197982135324349212
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1758865887090,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro
new file mode 100644
index 00000000000..1887220ebf0
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro
new file mode 100644
index 00000000000..7218d1326d3
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..160d000fed6
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..3ec61d58469
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..b0f56644ae5
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..dee411d7ef7
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..4264e841218
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..0264972b11d
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..3db75d8c01a
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
new file mode 100644
index 00000000000..4838bb7d6e7
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json
new file mode 100644
index 00000000000..e6fc794e81f
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json
@@ -0,0 +1,145 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "baf92432-1443-47fd-971a-0a9df2432e4a",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1758188914000,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json
new file mode 100644
index 00000000000..d52c580cbf2
--- /dev/null
+++ b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json
@@ -0,0 +1,176 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "baf92432-1443-47fd-971a-0a9df2432e4a",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1758188959726,
+ "last-column-id" : 18,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 4,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 8,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 9,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 10,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 11,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 5,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 14,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 15,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 6,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 16,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 1689809712605583552,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 1689809712605583552,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 1689809712605583552,
+ "timestamp-ms" : 1758188959726,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0668",
+ "added-data-files" : "8",
+ "added-records" : "10",
+ "added-files-size" : "32551",
+ "changed-partition-count" : "1",
+ "total-records" : "10",
+ "total-files-size" : "32551",
+ "total-data-files" : "8",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1758188959726,
+ "snapshot-id" : 1689809712605583552
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1758188914000,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/complex_user_profiles_iceberg/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro
new file mode 100644
index 00000000000..201c7908bc4
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro differ
diff --git a/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro
new file mode 100644
index 00000000000..1f6548d7ba8
Binary files /dev/null and b/be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc
new file mode 100644
index 00000000000..2f63195ff34
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json
new file mode 100644
index 00000000000..174f7a92c79
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json
@@ -0,0 +1,548 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "a93bcb6d-568f-4fc2-8235-dc8e218c1beb",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1760884376565,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json
new file mode 100644
index 00000000000..3fc692fea56
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json
@@ -0,0 +1,579 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "a93bcb6d-568f-4fc2-8235-dc8e218c1beb",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1760884424603,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "orc",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 514179801967160228,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 514179801967160228,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 514179801967160228,
+ "timestamp-ms" : 1760884424603,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0681",
+ "added-data-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "7181",
+ "changed-partition-count" : "1",
+ "total-records" : "1",
+ "total-files-size" : "7181",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1760884424603,
+ "snapshot-id" : 514179801967160228
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1760884376565,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro
new file mode 100644
index 00000000000..d8b578a8f86
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro
@@ -0,0 +1 @@
+Obj schema�]{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"profile","required":false,"type":{"type":"struct","fields":[{"id":9,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":12,"name":"street","required":false,"type":"string"},{"id":13,"name":"city","required":false,"type":"string"},{"id":14,"name":"coordinates","required":false,"type":{"type":"struc [...]
format-version2"partition-spec-id0 iceberg.schema�+{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position delet [...]
partition-spec[]contentdata v�7�l���r���P�Kq=��l��XY~,� ð,�5;3� *baia��6_�kێͮ"!!!�co��{�B=��l��}8�s� 8̇�����m7 `��bg�4{dx��N7�̕N ��j`�9Q�cmz��A�[�]� s��{��r.6[A���7�n���3}��Vo�����7\3�S6�5�j�V}�5�Vq+ިeX=i٥�;y~�o/��H�I�"-я ��s� T�I��GA�A �J�* �,��1��'�8&�����}�:�\��.箨���������"X��d[,OD��\����ܐw7 ��p���C�#J?&����H�"���/(�M�%�^��!�^����[��H{G�}ʾ'��>�~@\dNp��8łӬp?gXr�s<�*k�s� ��gEqHA *@9�@����\�a���ul��������봻9����ѵ���.�a0 �r����P��6��Q��T*ò-;+��am�i���ݜ2�iW�親,�=<rbnr�_��O s2�v�7�
�l���r�
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro
new file mode 100644
index 00000000000..7621edf3b7f
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet
new file mode 100644
index 00000000000..014ad377122
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet differ
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json
new file mode 100644
index 00000000000..8bf743851ff
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json
@@ -0,0 +1,548 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "d3b72ae1-4880-4c3e-9111-ca1af3cc9c7c",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1760884395135,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json
new file mode 100644
index 00000000000..de90e000742
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json
@@ -0,0 +1,579 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "d3b72ae1-4880-4c3e-9111-ca1af3cc9c7c",
+ "location" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1760884432817,
+ "last-column-id" : 92,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 2,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "profile",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 9,
+ "name" : "address",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 12,
+ "name" : "street",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 13,
+ "name" : "city",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 14,
+ "name" : "coordinates",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 15,
+ "name" : "lat",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 16,
+ "name" : "lng",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 10,
+ "name" : "contact",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 17,
+ "name" : "email",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 18,
+ "name" : "phone",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 19,
+ "name" : "country_code",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 20,
+ "name" : "number",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ }
+ }, {
+ "id" : 11,
+ "name" : "hobbies",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 21,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 22,
+ "name" : "name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 23,
+ "name" : "level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ }, {
+ "id" : 4,
+ "name" : "tags",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 24,
+ "element" : "string",
+ "element-required" : false
+ }
+ }, {
+ "id" : 5,
+ "name" : "friends",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 25,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 26,
+ "name" : "user_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 27,
+ "name" : "nickname",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 28,
+ "name" : "friendship_level",
+ "required" : false,
+ "type" : "int"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 6,
+ "name" : "recent_activity",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 29,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 30,
+ "name" : "action",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 31,
+ "name" : "details",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 32,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 33,
+ "name" : "key",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 34,
+ "name" : "value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 7,
+ "name" : "attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 35,
+ "key" : "string",
+ "value-id" : 36,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 8,
+ "name" : "complex_attributes",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 37,
+ "key" : "string",
+ "value-id" : 38,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 39,
+ "name" : "metadata",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 43,
+ "name" : "version",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 44,
+ "name" : "created_time",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 45,
+ "name" : "last_updated",
+ "required" : false,
+ "type" : "timestamptz"
+ }, {
+ "id" : 46,
+ "name" : "quality_score",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ }, {
+ "id" : 40,
+ "name" : "historical_scores",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 47,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 48,
+ "name" : "period",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 49,
+ "name" : "score",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 50,
+ "name" : "components",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 52,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 53,
+ "name" : "component_name",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 54,
+ "name" : "weight",
+ "required" : false,
+ "type" : "float"
+ }, {
+ "id" : 55,
+ "name" : "sub_scores",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 56,
+ "key" : "string",
+ "value-id" : 57,
+ "value" : "double",
+ "value-required" : false
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 51,
+ "name" : "trends",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 58,
+ "name" : "direction",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 59,
+ "name" : "magnitude",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 60,
+ "name" : "confidence_interval",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 61,
+ "name" : "lower",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 62,
+ "name" : "upper",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 41,
+ "name" : "hierarchical_data",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 63,
+ "key" : "string",
+ "value-id" : 64,
+ "value" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 65,
+ "name" : "category",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 66,
+ "name" : "sub_items",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 68,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 69,
+ "name" : "item_id",
+ "required" : false,
+ "type" : "long"
+ }, {
+ "id" : 70,
+ "name" : "properties",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 72,
+ "key" : "string",
+ "value-id" : 73,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 71,
+ "name" : "metrics",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 74,
+ "name" : "value",
+ "required" : false,
+ "type" : "double"
+ }, {
+ "id" : 75,
+ "name" : "unit",
+ "required" : false,
+ "type" : "string"
+ } ]
+ }
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 67,
+ "name" : "summary",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 76,
+ "name" : "total_count",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 77,
+ "name" : "average_value",
+ "required" : false,
+ "type" : "double"
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ }, {
+ "id" : 42,
+ "name" : "validation_rules",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 78,
+ "name" : "required",
+ "required" : false,
+ "type" : "boolean"
+ }, {
+ "id" : 79,
+ "name" : "constraints",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 81,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 82,
+ "name" : "rule_type",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 83,
+ "name" : "parameters",
+ "required" : false,
+ "type" : {
+ "type" : "map",
+ "key-id" : 85,
+ "key" : "string",
+ "value-id" : 86,
+ "value" : "string",
+ "value-required" : false
+ }
+ }, {
+ "id" : 84,
+ "name" : "error_message",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ }, {
+ "id" : 80,
+ "name" : "complex_constraint",
+ "required" : false,
+ "type" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 87,
+ "name" : "logic_operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 88,
+ "name" : "operands",
+ "required" : false,
+ "type" : {
+ "type" : "list",
+ "element-id" : 89,
+ "element" : {
+ "type" : "struct",
+ "fields" : [ {
+ "id" : 90,
+ "name" : "field_path",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 91,
+ "name" : "operator",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 92,
+ "name" : "comparison_value",
+ "required" : false,
+ "type" : "string"
+ } ]
+ },
+ "element-required" : false
+ }
+ } ]
+ }
+ } ]
+ }
+ } ]
+ },
+ "value-required" : false
+ }
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "owner" : "hive",
+ "write.format.default" : "parquet",
+ "write.parquet.compression-codec" : "zstd"
+ },
+ "current-snapshot-id" : 6622777947381868002,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 6622777947381868002,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 6622777947381868002,
+ "timestamp-ms" : 1760884432817,
+ "summary" : {
+ "operation" : "append",
+ "spark.app.id" : "application_1738810850199_0681",
+ "added-data-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "21586",
+ "changed-partition-count" : "1",
+ "total-records" : "1",
+ "total-files-size" : "21586",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "partition-statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1760884432817,
+ "snapshot-id" : 6622777947381868002
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1760884395135,
+ "metadata-file" : "hdfs://hdfs-cluster/user/hive/warehouse/cqtest.db/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro
new file mode 100644
index 00000000000..6071b684120
--- /dev/null
+++ b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro
@@ -0,0 +1,5 @@
+Obj schema�]{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"long"},{"id":2,"name":"name","required":false,"type":"string"},{"id":3,"name":"profile","required":false,"type":{"type":"struct","fields":[{"id":9,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":12,"name":"street","required":false,"type":"string"},{"id":13,"name":"city","required":false,"type":"string"},{"id":14,"name":"coordinates","required":false,"type":{"type":"struc [...]
format-version2"partition-spec-id0 iceberg.schema�+{"type":"struct","schema-id":0,"fields":[{"id":0,"name":"status","required":true,"type":"int"},{"id":1,"name":"snapshot_id","required":false,"type":"long"},{"id":3,"name":"sequence_number","required":false,"type":"long"},{"id":4,"name":"file_sequence_number","required":false,"type":"long"},{"id":2,"name":"data_file","required":true,"type":{"type":"struct","fields":[{"id":134,"name":"content","required":true,"type":"int","doc":"Contents of the file: 0=data, 1=position delet [...]
partition-spec[]contentdata ��b���}zd�ɤb����
+Œ�kaǟ���Lr���! AB�^ri��H�g��D*�?bC�K�br���rM%� �A�C)����!H)����M�:��(����{����wx����~����{�����߾�F xIe�T� :�X4V�fW�,�+ -��XZ���`q���v@-���Լ3�7�Z�bh�|��4�n�T��U����%Gb\T.iRh,R�H,$�%-,��HTT�qE��x�U$љN gg���Z��@��D:�ri�1\�䄜����K#] R;�G�Xaܘl'�qʚ��픑62F�X�
+��C�4_��m�Qm�-���`���:>� �`�M�na���m� ��a�]l�ak �|������lGn& �% �i� p�(�h�&�7EӔ��)CYZ����.�eS��Ԣ6u��H��m �C�D�B�#m# �v�v����������߁7 ����0� Q��8LB�`���4d �P 0�Є����EXG�@�!l"l!��;�{��GlShH`,�q� �泌�a� �;��㓵rUHԴ �+�
+�� 3J�*����MmMX�Y��ȗ �H�ٓ[�DŽ���x�ʝ�,�??� ν�
+:�Ԛ&k���(N�)��] ��H$"J!)d�> _�N�� 7x�3�������E�uR�����s������O�q~�_�����b���}zd�ɤb���
\ No newline at end of file
diff --git a/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro
new file mode 100644
index 00000000000..a3eee4e6a11
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro differ
diff --git a/be/test/exec/test_data/nested_user_profiles_orc/_SUCCESS b/be/test/exec/test_data/nested_user_profiles_orc/_SUCCESS
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc b/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc
new file mode 100644
index 00000000000..8a0f8bc17b4
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc differ
diff --git a/be/test/exec/test_data/nested_user_profiles_parquet/_SUCCESS b/be/test/exec/test_data/nested_user_profiles_parquet/_SUCCESS
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet b/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet
new file mode 100644
index 00000000000..73ae0061c21
Binary files /dev/null and b/be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet differ
diff --git a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
index 797ab9fc7db..edd131a2410 100644
--- a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
+++ b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp
@@ -140,20 +140,18 @@ TEST_F(ParquetThriftReaderTest, complex_nested_file) {
ASSERT_EQ(schemaDescriptor.get_column_index("hobby"), 2);
auto hobby = schemaDescriptor.get_column("hobby");
- // should be parsed as ARRAY<MAP<STRUCT<STRING,STRING>>>
- ASSERT_TRUE(hobby->children.size() == 1 && hobby->children[0].children.size() == 1 &&
- hobby->children[0].children[0].children.size() == 2);
+ // should be parsed as ARRAY<MAP<KEY,VALUE>>
+ ASSERT_TRUE(hobby->children.size() == 1 && hobby->children[0].children.size() == 2);
ASSERT_TRUE(hobby->data_type->get_primitive_type() == TYPE_ARRAY &&
- hobby->children[0].data_type->get_primitive_type() == TYPE_MAP &&
- hobby->children[0].children[0].data_type->get_primitive_type() == TYPE_STRUCT);
+ hobby->children[0].data_type->get_primitive_type() == TYPE_MAP);
// hobby(opt) --- bag(rep) --- array_element(opt) --- map(rep)
// \------- key(req)
// \------- value(opt)
// R=0,D=1 R=1,D=2 R=1,D=3 R=2,D=4
// \------ R=2,D=4
// \------ R=2,D=5
- auto h_key = hobby->children[0].children[0].children[0];
- auto h_value = hobby->children[0].children[0].children[1];
+ auto h_key = hobby->children[0];
+ auto h_value = hobby->children[1];
ASSERT_TRUE(h_key.repetition_level == 2 && h_key.definition_level == 4);
ASSERT_TRUE(h_value.repetition_level == 2 && h_value.definition_level == 5);
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
new file mode 100644
index 00000000000..7efca721fcd
--- /dev/null
+++ b/be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
@@ -0,0 +1,1179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+#include "vec/exec/format/table/hive_reader.h"
+
+namespace doris::vectorized {
+
+// Define the column access path configuration structure
+struct ColumnAccessPathConfig {
+ std::string column_name;
+ std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
+ std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
+ bool use_name_paths = true; // true=use field names, false=use indices
+};
+
+// ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+// 0: struct (table/root)
+// 1: id (bigint)
+// 2: name (string)
+// 3: profile (struct)
+// 4: address (struct)
+// 5: street (string)
+// 6: city (string)
+// 7: coordinates (struct)
+// 8: lat (double)
+// 9: lng (double)
+// 10: contact (struct)
+// 11: email (string)
+// 12: phone (struct)
+// 13: country_code (string)
+// 14: number (string)
+// 15: hobbies (array)
+// 16: element (struct)
+// 17: name (string)
+// 18: level (int)
+// 19: tags (array)
+// 20: element (string)
+// 21: friends (array)
+// 22: element (struct)
+// 23: user_id (bigint)
+// 24: nickname (string)
+// 25: friendship_level (int)
+// 26: recent_activity (array)
+// 27: element (struct)
+// 28: action (string)
+// 29: details (array)
+// 30: element (struct)
+// 31: key (string)
+// 32: value (string)
+// 33: attributes (map)
+// 34: key (string)
+// 35: value (string)
+// 36: complex_attributes (map)
+// 37: key (string)
+// 38: value (struct)
+// 39: metadata (struct)
+// 40: version (string)
+// 41: created_time (timestamp)
+// 42: last_updated (timestamp)
+// 43: quality_score (double)
+// 44: historical_scores (array)
+// 45: element (struct)
+// 46: period (string)
+// 47: score (double)
+// 48: components (array)
+// 49: element (struct)
+// 50: component_name (string)
+// 51: weight (float)
+// 52: sub_scores (map)
+// 53: key (string)
+// 54: value (double)
+// 55: trends (struct)
+// 56: direction (string)
+// 57: magnitude (double)
+// 58: confidence_interval (struct)
+// 59: lower (double)
+// 60: upper (double)
+// 61: hierarchical_data (map)
+// 62: key (string)
+// 63: value (struct)
+// 64: category (string)
+// 65: sub_items (array)
+// 66: element (struct)
+// 67: item_id (bigint)
+// 68: properties (map)
+// 69: key (string)
+// 70: value (string)
+// 71: metrics (struct)
+// 72: value (double)
+// 73: unit (string)
+// 74: summary (struct)
+// 75: total_count (int)
+// 76: average_value (double)
+// 77: validation_rules (struct)
+// 78: required (boolean)
+// 79: constraints (array)
+// 80: element (struct)
+// 81: rule_type (string)
+// 82: parameters (map)
+// 83: key (string)
+// 84: value (string)
+// 85: error_message (string)
+// 86: complex_constraint (struct)
+// 87: logic_operator (string)
+// 88: operands (array)
+// 89: element (struct)
+// 90: field_path (string)
+// 91: operator (string)
+// 92: comparison_value (string)
+
+class HiveReaderCreateColumnIdsTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types,
+ const std::vector<ColumnAccessPathConfig>& access_configs = {}) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [this, &access_configs](
+ TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::HIVE_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "id") {
+ // id: bigint
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::BIGINT);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "name") {
+ // name: string
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::STRING);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "tags") {
+ // tags: array<string>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_node;
+ element_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType element_scalar;
+ element_scalar.__set_type(TPrimitiveType::STRING);
+ element_node.__set_scalar_type(element_scalar);
+ type.types.push_back(element_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "friends") {
+ // friends: array<struct<user_id:bigint, nickname:string, friendship_level:int>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField user_id_field;
+ user_id_field.__set_name("user_id");
+ user_id_field.__set_contains_null(true);
+ element_fields.push_back(user_id_field);
+ TStructField nickname_field;
+ nickname_field.__set_name("nickname");
+ nickname_field.__set_contains_null(true);
+ element_fields.push_back(nickname_field);
+ TStructField friendship_level_field;
+ friendship_level_field.__set_name("friendship_level");
+ friendship_level_field.__set_contains_null(true);
+ element_fields.push_back(friendship_level_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // user_id: bigint
+ TTypeNode user_id_node;
+ user_id_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType user_id_scalar;
+ user_id_scalar.__set_type(TPrimitiveType::BIGINT);
+ user_id_node.__set_scalar_type(user_id_scalar);
+ type.types.push_back(user_id_node);
+ // nickname: string
+ TTypeNode nickname_node;
+ nickname_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType nickname_scalar;
+ nickname_scalar.__set_type(TPrimitiveType::STRING);
+ nickname_node.__set_scalar_type(nickname_scalar);
+ type.types.push_back(nickname_node);
+ // friendship_level: int
+ TTypeNode friendship_level_node;
+ friendship_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType friendship_level_scalar;
+ friendship_level_scalar.__set_type(TPrimitiveType::INT);
+ friendship_level_node.__set_scalar_type(friendship_level_scalar);
+ type.types.push_back(friendship_level_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "recent_activity") {
+ // recent_activity: array<struct<action:string, details:array<struct<key:string, value:string>>>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField action_field;
+ action_field.__set_name("action");
+ action_field.__set_contains_null(true);
+ element_fields.push_back(action_field);
+ TStructField details_field;
+ details_field.__set_name("details");
+ details_field.__set_contains_null(true);
+ element_fields.push_back(details_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // action: string
+ TTypeNode action_node;
+ action_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType action_scalar;
+ action_scalar.__set_type(TPrimitiveType::STRING);
+ action_node.__set_scalar_type(action_scalar);
+ type.types.push_back(action_node);
+ // details: array<struct<key:string, value:string>>
+ TTypeNode details_array_node;
+ details_array_node.__set_type(TTypeNodeType::ARRAY);
+ details_array_node.__set_contains_nulls({true});
+ type.types.push_back(details_array_node);
+ TTypeNode details_element_struct_node;
+ details_element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> details_element_fields;
+ TStructField key_field;
+ key_field.__set_name("key");
+ key_field.__set_contains_null(true);
+ details_element_fields.push_back(key_field);
+ TStructField value_field;
+ value_field.__set_name("value");
+ value_field.__set_contains_null(true);
+ details_element_fields.push_back(value_field);
+ details_element_struct_node.__set_struct_fields(details_element_fields);
+ type.types.push_back(details_element_struct_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "attributes") {
+ // attributes: map<string, string>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "complex_attributes") {
+ // complex_attributes: map<string, struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>
+ TTypeNode value_struct_node;
+ value_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> value_fields;
+ TStructField metadata_field;
+ metadata_field.__set_name("metadata");
+ metadata_field.__set_contains_null(true);
+ value_fields.push_back(metadata_field);
+ value_struct_node.__set_struct_fields(value_fields);
+ type.types.push_back(value_struct_node);
+ // metadata: struct<version:string, created_time:timestamp, last_updated:timestamp>
+ TTypeNode metadata_struct_node;
+ metadata_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> metadata_fields;
+ TStructField version_field;
+ version_field.__set_name("version");
+ version_field.__set_contains_null(true);
+ metadata_fields.push_back(version_field);
+ TStructField created_time_field;
+ created_time_field.__set_name("created_time");
+ created_time_field.__set_contains_null(true);
+ metadata_fields.push_back(created_time_field);
+ TStructField last_updated_field;
+ last_updated_field.__set_name("last_updated");
+ last_updated_field.__set_contains_null(true);
+ metadata_fields.push_back(last_updated_field);
+ metadata_struct_node.__set_struct_fields(metadata_fields);
+ type.types.push_back(metadata_struct_node);
+ // version: string
+ TTypeNode version_node;
+ version_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType version_scalar;
+ version_scalar.__set_type(TPrimitiveType::STRING);
+ version_node.__set_scalar_type(version_scalar);
+ type.types.push_back(version_node);
+ // created_time: timestamp
+ TTypeNode created_time_node;
+ created_time_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType created_time_scalar;
+ created_time_scalar.__set_type(TPrimitiveType::DATETIME);
+ created_time_node.__set_scalar_type(created_time_scalar);
+ type.types.push_back(created_time_node);
+ // last_updated: timestamp
+ TTypeNode last_updated_node;
+ last_updated_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType last_updated_scalar;
+ last_updated_scalar.__set_type(TPrimitiveType::DATETIME);
+ last_updated_node.__set_scalar_type(last_updated_scalar);
+ type.types.push_back(last_updated_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY nodes need contains_nulls, SCALAR nodes do not
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // Regular types
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+
+ // Use configuration to set column_access_paths
+ for (const auto& config : access_configs) {
+ set_column_access_paths(tslot_desc, config);
+ }
+
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function: Set column access paths
+ void set_column_access_paths(TSlotDescriptor& tslot_desc,
+ const ColumnAccessPathConfig& config) {
+ if (config.column_name != tslot_desc.colName) {
+ return; // Not the target column, skip
+ }
+
+ // Set all_column_access_paths
+ if (!config.all_column_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.all_column_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+
+ // Set predicate_column_access_paths
+ if (!config.predicate_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.predicate_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_predicate_access_paths(access_paths);
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+
+ // Helper function: Create and setup ParquetReader
+ std::tuple<std::unique_ptr<HiveParquetReader>, const FieldDescriptor*> create_parquet_reader(
+ const std::string& test_file) {
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ RuntimeState runtime_state((TQueryGlobals()));
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size();
+ scan_range.path = test_file;
+ RuntimeProfile profile("test_profile");
+
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+ auto generic_reader =
+ ParquetReader::create_unique(&profile, scan_params, scan_range, 1024, &ctz, nullptr,
+ &runtime_state, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ const FieldDescriptor* field_desc = nullptr;
+ st = parquet_reader->get_file_metadata_schema(&field_desc);
+ if (!st.ok() || !field_desc) {
+ return {nullptr, nullptr};
+ }
+
+ auto hive_reader = std::make_unique<HiveParquetReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(hive_reader), field_desc};
+ }
+
+ // Helper function: Create and setup OrcReader
+ std::tuple<std::unique_ptr<HiveOrcReader>, const orc::Type*> create_orc_reader(
+ const std::string& test_file) {
+ // Open the Hive Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ OrcReader::create_unique(&profile, &runtime_state, scan_params, scan_range, 1024,
+ "CST", nullptr, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto orc_reader = static_cast<OrcReader*>(generic_reader.get());
+ // Get FieldDescriptor from Orc file
+ const orc::Type* orc_type_ptr = nullptr;
+ st = orc_reader->get_file_type(&orc_type_ptr);
+ if (!st.ok() || !orc_type_ptr) {
+ return {nullptr, nullptr};
+ }
+
+ // Create HiveOrcReader
+ auto hive_reader = std::make_unique<HiveOrcReader>(std::move(generic_reader), &profile,
+ &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(hive_reader), orc_type_ptr};
+ }
+
+ // Helper function: Run Parquet test with different column ID extraction methods
+ void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_parquet/"
+ "part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet";
+
+ auto [hive_reader, field_desc] = create_parquet_reader(test_file);
+ if (!hive_reader || !field_desc) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ actual_result = HiveParquetReader::_create_column_ids_by_top_level_col_index(
+ field_desc, tuple_descriptor);
+ } else {
+ actual_result = HiveParquetReader::_create_column_ids(field_desc, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+
+ // Helper function: Run ORC test with different column ID extraction methods
+ void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_orc/"
+ "part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc";
+
+ auto [hive_reader, orc_type] = create_orc_reader(test_file);
+ if (!hive_reader || !orc_type) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ actual_result = HiveOrcReader::_create_column_ids_by_top_level_col_index(
+ orc_type, tuple_descriptor);
+ } else {
+ actual_result = HiveOrcReader::_create_column_ids(orc_type, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+};
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_1) {
+ // Properties:
+
+ // Configure access paths for profile column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "address", "coordinates", "lng"},
+ {"profile", "contact", "email"},
+ {"profile", "hobbies", "*", "level"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_2) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates", "lat"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_3) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC column IDs are assigned in a tree-increment order, root is 0 and children increment sequentially.
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
+ access_config.predicate_paths = {{"profile", "address", "coordinates"},
+ {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_4) {
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {};
+ access_config.predicate_paths = {};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids,
+ true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // 19: tags (array)
+ // 20: element (string)
+ // 21: friends (array)
+ // 22: element (struct)
+ // 23: user_id (bigint)
+ // 24: nickname (string)
+ // 25: friendship_level (int)
+ // 26: recent_activity (array)
+ // 27: element (struct)
+ // 28: action (string)
+ // 29: details (array)
+ // 30: element (struct)
+ // 31: key (string)
+ // 32: value (string)
+
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // Configure access paths for friends column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "friends";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"friends", "*", "nickname"},
+ {"friends", "*", "friendship_level"}};
+ access_config.predicate_paths = {
+ {"friends", "*", "nickname"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // Configure access paths for recent_activity column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "recent_activity";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"recent_activity", "*", "action"},
+ {"recent_activity", "*", "details", "*", "value"}};
+ access_config.predicate_paths = {
+ {"recent_activity", "*", "action"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids,
+ true);
+
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids, true);
+}
+
+TEST_F(HiveReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // 33: attributes (map)
+ // 34: key (string)
+ // 35: value (string)
+ // 36: complex_attributes (map)
+ // 37: key (string)
+ // 38: value (struct)
+ // 39: metadata (struct)
+ // 40: version (string)
+ // 41: created_time (timestamp)
+ // 42: last_updated (timestamp)
+ // 43: quality_score (double)
+ // 44: historical_scores (array)
+ // 45: element (struct)
+ // 46: period (string)
+ // 47: score (double)
+ // 48: components (array)
+ // 49: element (struct)
+ // 50: component_name (string)
+ // 51: weight (float)
+ // 52: sub_scores (map)
+ // 53: key (string)
+ // 54: value (double)
+ // 55: trends (struct)
+ // 56: direction (string)
+ // 57: magnitude (double)
+ // 58: confidence_interval (struct)
+ // 59: lower (double)
+ // 60: upper (double)
+ // 61: hierarchical_data (map)
+ // 62: key (string)
+ // 63: value (struct)
+ // 64: category (string)
+ // 65: sub_items (array)
+ // 66: element (struct)
+ // 67: item_id (bigint)
+ // 68: properties (map)
+ // 69: key (string)
+ // 70: value (string)
+ // 71: metrics (struct)
+ // 72: value (double)
+ // 73: unit (string)
+ // 74: summary (struct)
+ // 75: total_count (int)
+ // 76: average_value (double)
+ // 77: validation_rules (struct)
+ // 78: required (boolean)
+ // 79: constraints (array)
+ // 80: element (struct)
+ // 81: rule_type (string)
+ // 82: parameters (map)
+ // 83: key (string)
+ // 84: value (string)
+ // 85: error_message (string)
+ // 86: complex_constraint (struct)
+ // 87: logic_operator (string)
+ // 88: operands (array)
+ // 89: element (struct)
+ // 90: field_path (string)
+ // 91: operator (string)
+ // 92: comparison_value (string)
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // Configure access paths for complex_attributes column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "complex_attributes";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {
+ {"complex_attributes", "*", "metadata", "version"},
+ {"complex_attributes", "*", "historical_scores", "*", "components", "*",
+ "sub_scores"},
+ {"complex_attributes", "*", "hierarchical_data", "VALUES"},
+ {"complex_attributes", "VALUES", "validation_rules", "constraints", "*",
+ "parameters", "KEYS"}};
+ access_config.predicate_paths = {{"complex_attributes", "*", "metadata", "version"}
+
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // parquet values should access keys
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> parquet_expected_column_ids = {
+ 2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53, 54, 61, 62, 63, 64,
+ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> parquet_expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
+ parquet_expected_filter_column_ids);
+ run_parquet_test_with_method(access_configs, parquet_expected_column_ids,
+ parquet_expected_filter_column_ids, true);
+ }
+
+ {
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> orc_expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
+ 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
+ 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> orc_expected_filter_column_ids = {36, 37, 38, 39, 40};
+ run_orc_test_with_method(access_configs, orc_expected_column_ids,
+ orc_expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, orc_expected_column_ids,
+ orc_expected_filter_column_ids, true);
+ }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/exec/format/table/hive/hive_reader_test.cpp b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
new file mode 100644
index 00000000000..16bde5c7bae
--- /dev/null
+++ b/be/test/vec/exec/format/table/hive/hive_reader_test.cpp
@@ -0,0 +1,743 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/hive_reader.h"
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+
+namespace doris::vectorized::table {
+
+class HiveReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create complex struct types for testing
+ void create_complex_struct_types(DataTypePtr& coordinates_struct_type,
+ DataTypePtr& address_struct_type,
+ DataTypePtr& phone_struct_type,
+ DataTypePtr& contact_struct_type,
+ DataTypePtr& hobby_element_struct_type,
+ DataTypePtr& hobbies_array_type,
+ DataTypePtr& profile_struct_type, DataTypePtr& name_type) {
+ // Create name column type (direct field)
+ name_type = make_nullable(std::make_shared<DataTypeString>());
+
+ // First create coordinates struct type
+ std::vector<DataTypePtr> coordinates_types = {
+ make_nullable(std::make_shared<DataTypeFloat64>()), // lat (field ID 10)
+ make_nullable(std::make_shared<DataTypeFloat64>()) // lng (field ID 11)
+ };
+ std::vector<std::string> coordinates_names = {"lat", "lng"};
+ coordinates_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(coordinates_types, coordinates_names));
+
+ // Create address struct type (with street, city, coordinates)
+ std::vector<DataTypePtr> address_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // street (field ID 7)
+ make_nullable(std::make_shared<DataTypeString>()), // city (field ID 8)
+ coordinates_struct_type // coordinates (field ID 9)
+ };
+ std::vector<std::string> address_names = {"street", "city", "coordinates"};
+ address_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(address_types, address_names));
+
+ // Create phone struct type
+ std::vector<DataTypePtr> phone_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // country_code (field ID 14)
+ make_nullable(std::make_shared<DataTypeString>()) // number (field ID 15)
+ };
+ std::vector<std::string> phone_names = {"country_code", "number"};
+ phone_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(phone_types, phone_names));
+
+ // Create contact struct type (with email, phone)
+ std::vector<DataTypePtr> contact_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // email (field ID 12)
+ phone_struct_type // phone (field ID 13)
+ };
+ std::vector<std::string> contact_names = {"email", "phone"};
+ contact_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(contact_types, contact_names));
+
+ // Create hobby element struct type for array elements
+ std::vector<DataTypePtr> hobby_element_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // name (field ID 17)
+ make_nullable(std::make_shared<DataTypeInt32>()) // level (field ID 18)
+ };
+ std::vector<std::string> hobby_element_names = {"name", "level"};
+ hobby_element_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(hobby_element_types, hobby_element_names));
+
+ // Create hobbies array type
+ hobbies_array_type =
+ make_nullable(std::make_shared<DataTypeArray>(hobby_element_struct_type));
+
+ // Create complete profile struct type (with address, contact, hobbies)
+ std::vector<DataTypePtr> profile_types = {
+ address_struct_type, // address (field ID 4)
+ contact_struct_type, // contact (field ID 5)
+ hobbies_array_type // hobbies (field ID 6)
+ };
+ std::vector<std::string> profile_names = {"address", "contact", "hobbies"};
+ profile_struct_type =
+ make_nullable(std::make_shared<DataTypeStruct>(profile_types, profile_names));
+ }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [](TDescriptorTable& t_desc_table, TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::OLAP_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY nodes set contains_nulls; SCALAR nodes do not
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // Regular type
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+ // Set column_access_paths only for the profile field
+ if (table_column_names[i] == "profile") {
+ {
+ std::vector<TColumnAccessPath> access_paths;
+ // address.coordinates.lat
+ TColumnAccessPath path1;
+ path1.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path1;
+ data_path1.__set_path({"profile", "address", "coordinates", "lat"});
+ path1.__set_data_access_path(data_path1);
+ access_paths.push_back(path1);
+ // address.coordinates.lng
+ TColumnAccessPath path2;
+ path2.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path2;
+ data_path2.__set_path({"profile", "address", "coordinates", "lng"});
+ path2.__set_data_access_path(data_path2);
+ access_paths.push_back(path2);
+ // contact.email
+ TColumnAccessPath path3;
+ path3.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path3;
+ data_path3.__set_path({"profile", "contact", "email"});
+ path3.__set_data_access_path(data_path3);
+ access_paths.push_back(path3);
+ // hobbies[].element.level
+ TColumnAccessPath path4;
+ path4.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path4;
+ data_path4.__set_path({"profile", "hobbies", "*", "level"});
+ path4.__set_data_access_path(data_path4);
+ access_paths.push_back(path4);
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+ }
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function to recursively print column row counts and check size > 0
+ void verify_test_results(Block& block, size_t read_rows) {
+ // Verify that we read some data
+ EXPECT_GT(read_rows, 0) << "Should read at least one row";
+ EXPECT_EQ(block.rows(), read_rows);
+
+ // Verify column count matches expected (2 columns: name, profile)
+ EXPECT_EQ(block.columns(), 2);
+
+ // Verify column names and types
+ auto columns_with_names = block.get_columns_with_type_and_name();
+ std::vector<std::string> expected_column_names = {"name", "profile"};
+ for (size_t i = 0; i < expected_column_names.size(); i++) {
+ EXPECT_EQ(columns_with_names[i].name, expected_column_names[i]);
+ }
+
+ // Verify column types
+ EXPECT_TRUE(columns_with_names[0].type->get_name().find("String") !=
+ std::string::npos); // name is STRING
+ EXPECT_TRUE(columns_with_names[1].type->get_name().find("Struct") !=
+ std::string::npos); // profile is STRUCT
+
+ // Print row count for each column and nested subcolumns
+ std::cout << "Block rows: " << block.rows() << std::endl;
+
+ // Helper function to recursively print column row counts
+ std::function<void(const ColumnPtr&, const DataTypePtr&, const std::string&, int)>
+ print_column_rows = [&](const ColumnPtr& col, const DataTypePtr& type,
+ const std::string& name, int depth) {
+ std::string indent(depth * 2, ' ');
+ std::cout << indent << name << " row count: " << col->size() << std::endl;
+ EXPECT_GT(col->size(), 0) << name << " column/subcolumn size should be > 0";
+
+ // Check if it's a nullable column
+ if (const auto* nullable_col = typeid_cast<const ColumnNullable*>(col.get())) {
+ auto nested_type =
+ assert_cast<const DataTypeNullable*>(type.get())->get_nested_type();
+
+ // Only add ".nested" suffix for non-leaf (complex) nullable columns
+ // Leaf columns like String, Int, etc. should not get the ".nested" suffix
+ bool is_complex_type =
+ (typeid_cast<const DataTypeStruct*>(nested_type.get()) !=
+ nullptr) ||
+ (typeid_cast<const DataTypeArray*>(nested_type.get()) != nullptr) ||
+ (typeid_cast<const DataTypeMap*>(nested_type.get()) != nullptr);
+
+ std::string nested_name = is_complex_type ? name + ".nested" : name;
+ print_column_rows(nullable_col->get_nested_column_ptr(), nested_type,
+ nested_name, depth + (is_complex_type ? 1 : 0));
+ }
+ // Check if it's a struct column
+ else if (const auto* struct_col = typeid_cast<const ColumnStruct*>(col.get())) {
+ auto struct_type = assert_cast<const DataTypeStruct*>(type.get());
+ for (size_t i = 0; i < struct_col->tuple_size(); ++i) {
+ std::string field_name = struct_type->get_element_name(i);
+ auto field_type = struct_type->get_element(i);
+ print_column_rows(struct_col->get_column_ptr(i), field_type,
+ name + "." + field_name, depth + 1);
+ }
+ }
+ // Check if it's an array column
+ else if (const auto* array_col = typeid_cast<const ColumnArray*>(col.get())) {
+ auto array_type = assert_cast<const DataTypeArray*>(type.get());
+ auto element_type = array_type->get_nested_type();
+ print_column_rows(array_col->get_data_ptr(), element_type, name + ".data",
+ depth + 1);
+ }
+ };
+
+ // Print row counts for all columns
+ for (size_t i = 0; i < block.columns(); ++i) {
+ const auto& column_with_name = block.get_by_position(i);
+ print_column_rows(column_with_name.column, column_with_name.type, column_with_name.name,
+ 0);
+ EXPECT_EQ(column_with_name.column->size(), block.rows())
+ << "Column " << column_with_name.name << " size mismatch";
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+};
+
+// Test reading real Hive Parquet file using HiveTableReader
+TEST_F(HiveReaderTest, ReadHiveParquetFile) {
+ // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
+ // Setup table descriptor for test columns with new schema:
+ /**
+ Schema:
+ message table {
+ required int64 id = 1;
+ required binary name (STRING) = 2;
+ required group profile = 3 {
+ optional group address = 4 {
+ optional binary street (STRING) = 7;
+ optional binary city (STRING) = 8;
+ optional group coordinates = 9 {
+ optional double lat = 10;
+ optional double lng = 11;
+ }
+ }
+ optional group contact = 5 {
+ optional binary email (STRING) = 12;
+ optional group phone = 13 {
+ optional binary country_code (STRING) = 14;
+ optional binary number (STRING) = 15;
+ }
+ }
+ optional group hobbies (LIST) = 6 {
+ repeated group list {
+ optional group element = 16 {
+ optional binary name (STRING) = 17;
+ optional int32 level = 18;
+ }
+ }
+ }
+ }
+ }
+ */
+
+ // Open the Hive Parquet test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ std::string test_file =
+ "./be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/"
+ "00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet";
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ GTEST_SKIP() << "Test file not found: " << test_file;
+ return;
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create ParquetReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader = ParquetReader::create_unique(&profile, scan_params, scan_range, 1024,
+ &ctz, nullptr, &runtime_state, cache.get());
+ ASSERT_NE(generic_reader, nullptr);
+
+ // Set file reader for the generic reader
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ // Create HiveParquetReader
+ auto hive_reader = std::make_unique<HiveParquetReader>(std::move(generic_reader), &profile,
+ &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ // Create complex struct types using helper function
+ DataTypePtr coordinates_struct_type, address_struct_type, phone_struct_type;
+ DataTypePtr contact_struct_type, hobby_element_struct_type, hobbies_array_type;
+ DataTypePtr profile_struct_type, name_type;
+ create_complex_struct_types(coordinates_struct_type, address_struct_type, phone_struct_type,
+ contact_struct_type, hobby_element_struct_type, hobbies_array_type,
+ profile_struct_type, name_type);
+
+ // Create tuple descriptor using helper function
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+ std::vector<std::string> table_column_names = {"name", "profile"};
+ std::vector<int> table_column_positions = {1, 2};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ TPrimitiveType::STRING, TPrimitiveType::STRUCT // profile 用 STRUCT 类型
+ };
+ const TupleDescriptor* tuple_descriptor =
+ create_tuple_descriptor(&desc_tbl, obj_pool, t_desc_table, t_table_desc,
+ table_column_names, table_column_positions, table_column_types);
+
+ VExprContextSPtrs conjuncts; // Empty conjuncts for this test
+ std::vector<std::string> table_col_names = {"name", "profile"};
+ const std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range = nullptr;
+ const RowDescriptor* row_descriptor = nullptr;
+ const std::unordered_map<std::string, int>* colname_to_slot_id = nullptr;
+ const VExprContextSPtrs* not_single_slot_filter_conjuncts = nullptr;
+ const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts = nullptr;
+
+ st = hive_reader->init_reader(table_col_names, colname_to_value_range, conjuncts,
+ tuple_descriptor, row_descriptor, colname_to_slot_id,
+ not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+ ASSERT_TRUE(hive_reader->set_fill_columns(partition_columns, missing_columns).ok());
+
+ // Create block for reading nested structure (not flattened)
+ Block block;
+ {
+ MutableColumnPtr name_column = name_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(name_column), name_type, "name"));
+ // Add profile column (nested struct)
+ MutableColumnPtr profile_column = profile_struct_type->create_column();
+ block.insert(
+ ColumnWithTypeAndName(std::move(profile_column), profile_struct_type, "profile"));
+ }
+
+ // Read data from the file
+ size_t read_rows = 0;
+ bool eof = false;
+ st = hive_reader->get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // Verify test results using helper function
+ verify_test_results(block, read_rows);
+}
+
+// Test reading real Hive Orc file using HiveTableReader
+TEST_F(HiveReaderTest, ReadHiveOrcFile) {
+ // Read only: name, profile.address.coordinates.lat, profile.address.coordinates.lng, profile.contact.email
+ // Setup table descriptor for test columns with new schema:
+ /**
+ Schema:
+ message table {
+ required int64 id = 1;
+ required binary name (STRING) = 2;
+ required group profile = 3 {
+ optional group address = 4 {
+ optional binary street (STRING) = 7;
+ optional binary city (STRING) = 8;
+ optional group coordinates = 9 {
+ optional double lat = 10;
+ optional double lng = 11;
+ }
+ }
+ optional group contact = 5 {
+ optional binary email (STRING) = 12;
+ optional group phone = 13 {
+ optional binary country_code (STRING) = 14;
+ optional binary number (STRING) = 15;
+ }
+ }
+ optional group hobbies (LIST) = 6 {
+ repeated group list {
+ optional group element = 16 {
+ optional binary name (STRING) = 17;
+ optional int32 level = 18;
+ }
+ }
+ }
+ }
+ }
+ */
+ // Open the Hive Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ std::string test_file =
+ "./be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/"
+ "00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc";
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ GTEST_SKIP() << "Test file not found: " << test_file;
+ return;
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader = OrcReader::create_unique(&profile, &runtime_state, scan_params,
+ scan_range, 1024, "CST", nullptr, cache.get());
+ ASSERT_NE(generic_reader, nullptr);
+
+ // Create HiveOrcReader
+ auto hive_reader =
+ std::make_unique<HiveOrcReader>(std::move(generic_reader), &profile, &runtime_state,
+ scan_params, scan_range, nullptr, nullptr, cache.get());
+
+ // Create complex struct types using helper function
+ DataTypePtr coordinates_struct_type, address_struct_type, phone_struct_type;
+ DataTypePtr contact_struct_type, hobby_element_struct_type, hobbies_array_type;
+ DataTypePtr profile_struct_type, name_type;
+ create_complex_struct_types(coordinates_struct_type, address_struct_type, phone_struct_type,
+ contact_struct_type, hobby_element_struct_type, hobbies_array_type,
+ profile_struct_type, name_type);
+
+ // Create tuple descriptor using helper function
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+ std::vector<std::string> table_column_names = {"name", "profile"};
+ std::vector<int> table_column_positions = {1, 2};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ TPrimitiveType::STRING, TPrimitiveType::STRUCT // profile 用 STRUCT 类型
+ };
+ const TupleDescriptor* tuple_descriptor =
+ create_tuple_descriptor(&desc_tbl, obj_pool, t_desc_table, t_table_desc,
+ table_column_names, table_column_positions, table_column_types);
+
+ VExprContextSPtrs conjuncts; // Empty conjuncts for this test
+ std::vector<std::string> table_col_names = {"name", "profile"};
+ const std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range = nullptr;
+ const RowDescriptor* row_descriptor = nullptr;
+ const VExprContextSPtrs* not_single_slot_filter_conjuncts = nullptr;
+ const std::unordered_map<int, VExprContextSPtrs>* slot_id_to_filter_conjuncts = nullptr;
+
+ st = hive_reader->init_reader(table_col_names, colname_to_value_range, conjuncts,
+ tuple_descriptor, row_descriptor,
+ not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
+ ASSERT_TRUE(st.ok()) << st;
+
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> missing_columns;
+ ASSERT_TRUE(hive_reader->set_fill_columns(partition_columns, missing_columns).ok());
+
+ // Create block for reading nested structure (not flattened)
+ Block block;
+
+ {
+ MutableColumnPtr name_column = name_type->create_column();
+ block.insert(ColumnWithTypeAndName(std::move(name_column), name_type, "name"));
+ // Add profile column (nested struct)
+ MutableColumnPtr profile_column = profile_struct_type->create_column();
+ block.insert(
+ ColumnWithTypeAndName(std::move(profile_column), profile_struct_type, "profile"));
+ }
+
+ // Read data from the file
+ size_t read_rows = 0;
+ bool eof = false;
+ st = hive_reader->get_next_block(&block, &read_rows, &eof);
+ ASSERT_TRUE(st.ok()) << st;
+
+ // Verify test results using helper function
+ verify_test_results(block, read_rows);
+}
+
+} // namespace doris::vectorized::table
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
new file mode 100644
index 00000000000..47e0ad87a4a
--- /dev/null
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
@@ -0,0 +1,1177 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+#include "vec/exec/format/table/iceberg_reader.h"
+
+namespace doris::vectorized {
+
+// Define the column access path configuration structure
+struct ColumnAccessPathConfig {
+ std::string column_name;
+ std::vector<std::vector<std::string>> all_column_paths; // For all_column_access_paths
+ std::vector<std::vector<std::string>> predicate_paths; // For predicate_column_access_paths
+ bool use_name_paths = true; // true=use field names, false=use indices
+};
+
+// Iceberg 列ID分配 (基于 Protobuf Schema 定义)
+// 1: id (int64)
+// 2: name (string)
+// 3: profile (struct)
+// 9: address (struct)
+// 12: street (string)
+// 13: city (string)
+// 14: coordinates (struct)
+// 15: lat (double)
+// 16: lng (double)
+// 10: contact (struct)
+// 17: email (string)
+// 18: phone (struct)
+// 19: country_code (string)
+// 20: number (string)
+// 11: hobbies (list)
+// 21: element (struct)
+// 22: name (string)
+// 23: level (int32)
+// 4: tags (list)
+// 24: element (string)
+// 5: friends (list)
+// 25: element (struct)
+// 26: user_id (int64)
+// 27: nickname (string)
+// 28: friendship_level (int32)
+// 6: recent_activity (list)
+// 29: element (struct)
+// 30: action (string)
+// 31: details (list)
+// 32: element (struct)
+// 33: key (string)
+// 34: value (string)
+// 7: attributes (map)
+// 35: key (string)
+// 36: value (string)
+// 8: complex_attributes (map)
+// 37: key (string)
+// 38: value (struct)
+// 39: metadata (struct)
+// 43: version (string)
+// 44: created_time (timestamp)
+// 45: last_updated (timestamp)
+// 46: quality_score (double)
+// 40: historical_scores (list)
+// 47: element (struct)
+// 48: period (string)
+// 49: score (double)
+// 50: components (list)
+// 52: element (struct)
+// 53: component_name (string)
+// 54: weight (float)
+// 55: sub_scores (map)
+// 56: key (string)
+// 57: value (double)
+// 51: trends (struct)
+// 58: direction (string)
+// 59: magnitude (double)
+// 60: confidence_interval (struct)
+// 61: lower (double)
+// 62: upper (double)
+// 41: hierarchical_data (map)
+// 63: key (string)
+// 64: value (struct)
+// 65: category (string)
+// 66: sub_items (list)
+// 68: element (struct)
+// 69: item_id (int64)
+// 70: properties (map)
+// 72: key (string)
+// 73: value (string)
+// 71: metrics (struct)
+// 74: value (double)
+// 75: unit (string)
+// 67: summary (struct)
+// 76: total_count (int32)
+// 77: average_value (double)
+// 42: validation_rules (struct)
+// 78: required (boolean)
+// 79: constraints (list)
+// 81: element (struct)
+// 82: rule_type (string)
+// 83: parameters (map)
+// 85: key (string)
+// 86: value (string)
+// 84: error_message (string)
+// 80: complex_constraint (struct)
+// 87: logic_operator (string)
+// 88: operands (list)
+// 89: element (struct)
+// 90: field_path (string)
+// 91: operator (string)
+// 92: comparison_value (string)
+
+class IcebergReaderCreateColumnIdsTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to map column name to Iceberg field ID
+ int get_iceberg_field_id(const std::string& column_name) {
+ // Mapping based on the Iceberg schema definition in comments
+ static const std::unordered_map<std::string, int> column_to_field_id = {
+ {"id", 1}, {"name", 2},
+ {"profile", 3}, {"tags", 4},
+ {"friends", 5}, {"recent_activity", 6},
+ {"attributes", 7}, {"complex_attributes", 8}};
+
+ auto it = column_to_field_id.find(column_name);
+ if (it != column_to_field_id.end()) {
+ return it->second;
+ }
+ return -1; // Invalid field ID
+ }
+
+ // Helper function to create tuple descriptor
+ const TupleDescriptor* create_tuple_descriptor(
+ DescriptorTbl** desc_tbl, ObjectPool& obj_pool, TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc, const std::vector<std::string>& column_names,
+ const std::vector<int>& column_positions,
+ const std::vector<TPrimitiveType::type>& column_types,
+ const std::vector<ColumnAccessPathConfig>& access_configs = {}) {
+ // Create table descriptor with complex schema
+ auto create_table_desc = [this, &access_configs](
+ TDescriptorTable& t_desc_table,
+ TTableDescriptor& t_table_desc,
+ const std::vector<std::string>& table_column_names,
+ const std::vector<int>& table_column_positions,
+ const std::vector<TPrimitiveType::type>& types) {
+ t_table_desc.__set_id(0);
+ t_table_desc.__set_tableType(TTableType::HIVE_TABLE);
+ t_table_desc.__set_numCols(0);
+ t_table_desc.__set_numClusteringCols(0);
+ t_desc_table.tableDescriptors.push_back(t_table_desc);
+ t_desc_table.__isset.tableDescriptors = true;
+ for (int i = 0; i < table_column_names.size(); i++) {
+ TSlotDescriptor tslot_desc;
+ TTypeDesc type;
+ if (table_column_names[i] == "id") {
+ // id: bigint
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::BIGINT);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "name") {
+ // name: string
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(TPrimitiveType::STRING);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "tags") {
+ // tags: array<string>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_node;
+ element_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType element_scalar;
+ element_scalar.__set_type(TPrimitiveType::STRING);
+ element_node.__set_scalar_type(element_scalar);
+ type.types.push_back(element_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "friends") {
+ // friends: array<struct<user_id:bigint, nickname:string, friendship_level:int>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField user_id_field;
+ user_id_field.__set_name("user_id");
+ user_id_field.__set_contains_null(true);
+ element_fields.push_back(user_id_field);
+ TStructField nickname_field;
+ nickname_field.__set_name("nickname");
+ nickname_field.__set_contains_null(true);
+ element_fields.push_back(nickname_field);
+ TStructField friendship_level_field;
+ friendship_level_field.__set_name("friendship_level");
+ friendship_level_field.__set_contains_null(true);
+ element_fields.push_back(friendship_level_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // user_id: bigint
+ TTypeNode user_id_node;
+ user_id_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType user_id_scalar;
+ user_id_scalar.__set_type(TPrimitiveType::BIGINT);
+ user_id_node.__set_scalar_type(user_id_scalar);
+ type.types.push_back(user_id_node);
+ // nickname: string
+ TTypeNode nickname_node;
+ nickname_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType nickname_scalar;
+ nickname_scalar.__set_type(TPrimitiveType::STRING);
+ nickname_node.__set_scalar_type(nickname_scalar);
+ type.types.push_back(nickname_node);
+ // friendship_level: int
+ TTypeNode friendship_level_node;
+ friendship_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType friendship_level_scalar;
+ friendship_level_scalar.__set_type(TPrimitiveType::INT);
+ friendship_level_node.__set_scalar_type(friendship_level_scalar);
+ type.types.push_back(friendship_level_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "recent_activity") {
+ // recent_activity: array<struct<action:string, details:array<struct<key:string, value:string>>>>
+ TTypeNode array_node;
+ array_node.__set_type(TTypeNodeType::ARRAY);
+ array_node.__set_contains_nulls({true});
+ type.types.push_back(array_node);
+ TTypeNode element_struct_node;
+ element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> element_fields;
+ TStructField action_field;
+ action_field.__set_name("action");
+ action_field.__set_contains_null(true);
+ element_fields.push_back(action_field);
+ TStructField details_field;
+ details_field.__set_name("details");
+ details_field.__set_contains_null(true);
+ element_fields.push_back(details_field);
+ element_struct_node.__set_struct_fields(element_fields);
+ type.types.push_back(element_struct_node);
+ // action: string
+ TTypeNode action_node;
+ action_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType action_scalar;
+ action_scalar.__set_type(TPrimitiveType::STRING);
+ action_node.__set_scalar_type(action_scalar);
+ type.types.push_back(action_node);
+ // details: array<struct<key:string, value:string>>
+ TTypeNode details_array_node;
+ details_array_node.__set_type(TTypeNodeType::ARRAY);
+ details_array_node.__set_contains_nulls({true});
+ type.types.push_back(details_array_node);
+ TTypeNode details_element_struct_node;
+ details_element_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> details_element_fields;
+ TStructField key_field;
+ key_field.__set_name("key");
+ key_field.__set_contains_null(true);
+ details_element_fields.push_back(key_field);
+ TStructField value_field;
+ value_field.__set_name("value");
+ value_field.__set_contains_null(true);
+ details_element_fields.push_back(value_field);
+ details_element_struct_node.__set_struct_fields(details_element_fields);
+ type.types.push_back(details_element_struct_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "attributes") {
+ // attributes: map<string, string>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: string
+ TTypeNode value_node;
+ value_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType value_scalar;
+ value_scalar.__set_type(TPrimitiveType::STRING);
+ value_node.__set_scalar_type(value_scalar);
+ type.types.push_back(value_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "complex_attributes") {
+ // complex_attributes: map<string, struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>>
+ TTypeNode map_node;
+ map_node.__set_type(TTypeNodeType::MAP);
+ map_node.__set_contains_nulls({true, true});
+ type.types.push_back(map_node);
+ // key: string
+ TTypeNode key_node;
+ key_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType key_scalar;
+ key_scalar.__set_type(TPrimitiveType::STRING);
+ key_node.__set_scalar_type(key_scalar);
+ type.types.push_back(key_node);
+ // value: struct<metadata:struct<version:string, created_time:timestamp, last_updated:timestamp>>
+ TTypeNode value_struct_node;
+ value_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> value_fields;
+ TStructField metadata_field;
+ metadata_field.__set_name("metadata");
+ metadata_field.__set_contains_null(true);
+ value_fields.push_back(metadata_field);
+ value_struct_node.__set_struct_fields(value_fields);
+ type.types.push_back(value_struct_node);
+ // metadata: struct<version:string, created_time:timestamp, last_updated:timestamp>
+ TTypeNode metadata_struct_node;
+ metadata_struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> metadata_fields;
+ TStructField version_field;
+ version_field.__set_name("version");
+ version_field.__set_contains_null(true);
+ metadata_fields.push_back(version_field);
+ TStructField created_time_field;
+ created_time_field.__set_name("created_time");
+ created_time_field.__set_contains_null(true);
+ metadata_fields.push_back(created_time_field);
+ TStructField last_updated_field;
+ last_updated_field.__set_name("last_updated");
+ last_updated_field.__set_contains_null(true);
+ metadata_fields.push_back(last_updated_field);
+ metadata_struct_node.__set_struct_fields(metadata_fields);
+ type.types.push_back(metadata_struct_node);
+ // version: string
+ TTypeNode version_node;
+ version_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType version_scalar;
+ version_scalar.__set_type(TPrimitiveType::STRING);
+ version_node.__set_scalar_type(version_scalar);
+ type.types.push_back(version_node);
+ // created_time: timestamp
+ TTypeNode created_time_node;
+ created_time_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType created_time_scalar;
+ created_time_scalar.__set_type(TPrimitiveType::DATETIME);
+ created_time_node.__set_scalar_type(created_time_scalar);
+ type.types.push_back(created_time_node);
+ // last_updated: timestamp
+ TTypeNode last_updated_node;
+ last_updated_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType last_updated_scalar;
+ last_updated_scalar.__set_type(TPrimitiveType::DATETIME);
+ last_updated_node.__set_scalar_type(last_updated_scalar);
+ type.types.push_back(last_updated_node);
+ tslot_desc.__set_slotType(type);
+ } else if (table_column_names[i] == "profile") {
+ // STRUCT/ARRAY 节点设置 contains_nulls，SCALAR 节点不设置
+ TTypeNode struct_node;
+ struct_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> struct_fields;
+ TStructField address_field;
+ address_field.__set_name("address");
+ address_field.__set_contains_null(true);
+ struct_fields.push_back(address_field);
+ TStructField contact_field;
+ contact_field.__set_name("contact");
+ contact_field.__set_contains_null(true);
+ struct_fields.push_back(contact_field);
+ TStructField hobbies_field;
+ hobbies_field.__set_name("hobbies");
+ hobbies_field.__set_contains_null(true);
+ struct_fields.push_back(hobbies_field);
+ struct_node.__set_struct_fields(struct_fields);
+ type.types.push_back(struct_node);
+ TTypeNode address_node;
+ address_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> address_fields;
+ TStructField street_field;
+ street_field.__set_name("street");
+ street_field.__set_contains_null(true);
+ address_fields.push_back(street_field);
+ TStructField city_field;
+ city_field.__set_name("city");
+ city_field.__set_contains_null(true);
+ address_fields.push_back(city_field);
+ TStructField coordinates_field;
+ coordinates_field.__set_name("coordinates");
+ coordinates_field.__set_contains_null(true);
+ address_fields.push_back(coordinates_field);
+ address_node.__set_struct_fields(address_fields);
+ type.types.push_back(address_node);
+ TTypeNode street_node;
+ street_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType street_scalar;
+ street_scalar.__set_type(TPrimitiveType::STRING);
+ street_node.__set_scalar_type(street_scalar);
+ type.types.push_back(street_node);
+ TTypeNode city_node;
+ city_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType city_scalar;
+ city_scalar.__set_type(TPrimitiveType::STRING);
+ city_node.__set_scalar_type(city_scalar);
+ type.types.push_back(city_node);
+ TTypeNode coordinates_node;
+ coordinates_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> coordinates_fields;
+ TStructField lat_field;
+ lat_field.__set_name("lat");
+ lat_field.__set_contains_null(true);
+ coordinates_fields.push_back(lat_field);
+ TStructField lng_field;
+ lng_field.__set_name("lng");
+ lng_field.__set_contains_null(true);
+ coordinates_fields.push_back(lng_field);
+ coordinates_node.__set_struct_fields(coordinates_fields);
+ type.types.push_back(coordinates_node);
+ TTypeNode lat_node;
+ lat_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lat_scalar;
+ lat_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lat_node.__set_scalar_type(lat_scalar);
+ type.types.push_back(lat_node);
+ TTypeNode lng_node;
+ lng_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType lng_scalar;
+ lng_scalar.__set_type(TPrimitiveType::DOUBLE);
+ lng_node.__set_scalar_type(lng_scalar);
+ type.types.push_back(lng_node);
+ TTypeNode contact_node;
+ contact_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> contact_fields;
+ TStructField email_field;
+ email_field.__set_name("email");
+ email_field.__set_contains_null(true);
+ contact_fields.push_back(email_field);
+ TStructField phone_field;
+ phone_field.__set_name("phone");
+ phone_field.__set_contains_null(true);
+ contact_fields.push_back(phone_field);
+ contact_node.__set_struct_fields(contact_fields);
+ type.types.push_back(contact_node);
+ TTypeNode email_node;
+ email_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType email_scalar;
+ email_scalar.__set_type(TPrimitiveType::STRING);
+ email_node.__set_scalar_type(email_scalar);
+ type.types.push_back(email_node);
+ TTypeNode phone_node;
+ phone_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> phone_fields;
+ TStructField country_code_field;
+ country_code_field.__set_name("country_code");
+ country_code_field.__set_contains_null(true);
+ phone_fields.push_back(country_code_field);
+ TStructField number_field;
+ number_field.__set_name("number");
+ number_field.__set_contains_null(true);
+ phone_fields.push_back(number_field);
+ phone_node.__set_struct_fields(phone_fields);
+ type.types.push_back(phone_node);
+ TTypeNode country_code_node;
+ country_code_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType country_code_scalar;
+ country_code_scalar.__set_type(TPrimitiveType::STRING);
+ country_code_node.__set_scalar_type(country_code_scalar);
+ type.types.push_back(country_code_node);
+ TTypeNode number_node;
+ number_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType number_scalar;
+ number_scalar.__set_type(TPrimitiveType::STRING);
+ number_node.__set_scalar_type(number_scalar);
+ type.types.push_back(number_node);
+ TTypeNode hobbies_node;
+ hobbies_node.__set_type(TTypeNodeType::ARRAY);
+ hobbies_node.__set_contains_nulls({true});
+ type.types.push_back(hobbies_node);
+ TTypeNode hobby_element_node;
+ hobby_element_node.__set_type(TTypeNodeType::STRUCT);
+ std::vector<TStructField> hobby_element_fields;
+ TStructField hobby_name_field;
+ hobby_name_field.__set_name("name");
+ hobby_name_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_name_field);
+ TStructField hobby_level_field;
+ hobby_level_field.__set_name("level");
+ hobby_level_field.__set_contains_null(true);
+ hobby_element_fields.push_back(hobby_level_field);
+ hobby_element_node.__set_struct_fields(hobby_element_fields);
+ type.types.push_back(hobby_element_node);
+ TTypeNode hobby_name_node;
+ hobby_name_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_name_scalar;
+ hobby_name_scalar.__set_type(TPrimitiveType::STRING);
+ hobby_name_node.__set_scalar_type(hobby_name_scalar);
+ type.types.push_back(hobby_name_node);
+ TTypeNode hobby_level_node;
+ hobby_level_node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType hobby_level_scalar;
+ hobby_level_scalar.__set_type(TPrimitiveType::INT);
+ hobby_level_node.__set_scalar_type(hobby_level_scalar);
+ type.types.push_back(hobby_level_node);
+ tslot_desc.__set_slotType(type);
+ } else {
+ // 普通类型
+ TTypeNode node;
+ node.__set_type(TTypeNodeType::SCALAR);
+ TScalarType scalar_type;
+ scalar_type.__set_type(types[i]);
+ node.__set_scalar_type(scalar_type);
+ type.types.push_back(node);
+ tslot_desc.__set_slotType(type);
+ }
+ tslot_desc.__set_id(i);
+ tslot_desc.__set_parent(0);
+ tslot_desc.__set_colName(table_column_names[i]);
+ tslot_desc.__set_columnPos(table_column_positions[i]);
+ tslot_desc.__set_byteOffset(0);
+ tslot_desc.__set_nullIndicatorByte(0);
+ tslot_desc.__set_nullIndicatorBit(-1);
+ tslot_desc.__set_slotIdx(0);
+ tslot_desc.__set_isMaterialized(true);
+
+ // Set Iceberg field ID (col_unique_id)
+ int iceberg_field_id = get_iceberg_field_id(table_column_names[i]);
+ if (iceberg_field_id > 0) {
+ tslot_desc.__set_col_unique_id(iceberg_field_id);
+ }
+
+ // Use configuration to set column_access_paths
+ for (const auto& config : access_configs) {
+ set_column_access_paths(tslot_desc, config);
+ }
+
+ t_desc_table.slotDescriptors.push_back(tslot_desc);
+ }
+ t_desc_table.__isset.slotDescriptors = true;
+ TTupleDescriptor t_tuple_desc;
+ t_tuple_desc.__set_id(0);
+ t_tuple_desc.__set_byteSize(16);
+ t_tuple_desc.__set_numNullBytes(0);
+ t_tuple_desc.__set_tableId(0);
+ t_tuple_desc.__isset.tableId = true;
+ t_desc_table.tupleDescriptors.push_back(t_tuple_desc);
+ };
+
+ create_table_desc(t_desc_table, t_table_desc, column_names, column_positions, column_types);
+ EXPECT_TRUE(DescriptorTbl::create(&obj_pool, t_desc_table, desc_tbl).ok());
+ return (*desc_tbl)->get_tuple_descriptor(0);
+ }
+
+ // Helper function: set column access paths on a slot descriptor
+ void set_column_access_paths(TSlotDescriptor& tslot_desc,
+ const ColumnAccessPathConfig& config) {
+ if (config.column_name != tslot_desc.colName) {
+ return; // Not the target column, skip
+ }
+
+ // Set all_column_access_paths
+ if (!config.all_column_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.all_column_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_all_access_paths(access_paths);
+ }
+
+ // Set predicate_column_access_paths
+ if (!config.predicate_paths.empty()) {
+ std::vector<TColumnAccessPath> access_paths;
+ for (const auto& path_vector : config.predicate_paths) {
+ TColumnAccessPath access_path;
+ access_path.__set_type(doris::TAccessPathType::DATA);
+ TDataAccessPath data_path;
+ data_path.__set_path(path_vector);
+ access_path.__set_data_access_path(data_path);
+ access_paths.push_back(access_path);
+ }
+ tslot_desc.__set_predicate_access_paths(access_paths);
+ }
+ }
+
+ std::unique_ptr<doris::FileMetaCache> cache;
+ cctz::time_zone timezone_obj;
+
+ // Helper function: create and setup ParquetReader
+ std::tuple<std::unique_ptr<IcebergParquetReader>, const FieldDescriptor*> create_parquet_reader(
+ const std::string& test_file) {
+ // Open the Iceberg Parquet test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_PARQUET;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create ParquetReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ ParquetReader::create_unique(&profile, scan_params, scan_range, 1024, &ctz, nullptr,
+ &runtime_state, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ // Set file reader for the generic reader
+ auto parquet_reader = static_cast<ParquetReader*>(generic_reader.get());
+ parquet_reader->set_file_reader(file_reader);
+
+ const FieldDescriptor* field_desc = nullptr;
+ st = parquet_reader->get_file_metadata_schema(&field_desc);
+ if (!st.ok() || !field_desc) {
+ return {nullptr, nullptr};
+ }
+
+ // Create IcebergParquetReader
+ auto iceberg_reader = std::make_unique<IcebergParquetReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(iceberg_reader), field_desc};
+ }
+
+ // Helper function: create and setup OrcReader
+ std::tuple<std::unique_ptr<IcebergOrcReader>, const orc::Type*> create_orc_reader(
+ const std::string& test_file) {
+ // Open the Iceberg Orc test file
+ auto local_fs = io::global_local_filesystem();
+ io::FileReaderSPtr file_reader;
+ auto st = local_fs->open_file(test_file, &file_reader);
+ if (!st.ok()) {
+ return {nullptr, nullptr};
+ }
+
+ // Setup runtime state
+ RuntimeState runtime_state((TQueryGlobals()));
+
+ // Setup scan parameters
+ TFileScanRangeParams scan_params;
+ scan_params.format_type = TFileFormatType::FORMAT_ORC;
+
+ TFileRangeDesc scan_range;
+ scan_range.start_offset = 0;
+ scan_range.size = file_reader->size(); // Read entire file
+ scan_range.path = test_file;
+
+ // Create mock profile
+ RuntimeProfile profile("test_profile");
+
+ // Create OrcReader as the underlying file format reader
+ cctz::time_zone ctz;
+ TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
+
+ auto generic_reader =
+ OrcReader::create_unique(&profile, &runtime_state, scan_params, scan_range, 1024,
+ "CST", nullptr, cache.get());
+ if (!generic_reader) {
+ return {nullptr, nullptr};
+ }
+
+ auto orc_reader = static_cast<OrcReader*>(generic_reader.get());
+ // Get FieldDescriptor from Orc file
+ const orc::Type* orc_type_ptr = nullptr;
+ st = orc_reader->get_file_type(&orc_type_ptr);
+ if (!st.ok() || !orc_type_ptr) {
+ return {nullptr, nullptr};
+ }
+
+ // Create IcebergOrcReader
+ auto iceberg_reader = std::make_unique<IcebergOrcReader>(
+ std::move(generic_reader), &profile, &runtime_state, scan_params, scan_range,
+ nullptr, nullptr, cache.get());
+
+ return {std::move(iceberg_reader), orc_type_ptr};
+ }
+
+ // Helper function: run Parquet test with different column ID extraction methods
+ void run_parquet_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/"
+ "00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet";
+
+ auto [iceberg_reader, field_desc] = create_parquet_reader(test_file);
+ if (!iceberg_reader || !field_desc) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ // actual_result = IcebergParquetReader::_create_column_ids_by_top_level_col_index(
+ // field_desc, tuple_descriptor);
+ } else {
+ actual_result = IcebergParquetReader::_create_column_ids(field_desc, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+
+ // Helper function: run Orc test with different column ID extraction methods
+ void run_orc_test_with_method(const std::vector<ColumnAccessPathConfig>& access_configs,
+ const std::set<uint64_t>& expected_column_ids,
+ const std::set<uint64_t>& expected_filter_column_ids,
+ bool use_top_level_method = false,
+ bool should_skip_assertion = false) {
+ std::string test_file =
+ "./be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/"
+ "00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc";
+
+ auto [iceberg_reader, orc_type] = create_orc_reader(test_file);
+ if (!iceberg_reader || !orc_type) {
+ GTEST_SKIP() << "Test file not found or failed to create reader: " << test_file;
+ return;
+ }
+
+ // Create tuple descriptor based on full schema
+ DescriptorTbl* desc_tbl;
+ ObjectPool obj_pool;
+ TDescriptorTable t_desc_table;
+ TTableDescriptor t_table_desc;
+
+ // Define all columns according to the schema
+ std::vector<std::string> table_column_names = {
+ "name", "profile", "tags", "friends", "recent_activity",
+ "attributes", "complex_attributes"};
+ std::vector<int> table_column_positions = {1, 2, 3, 4, 5, 6, 7};
+ std::vector<TPrimitiveType::type> table_column_types = {
+ // TPrimitiveType::BIGINT, // id
+ TPrimitiveType::STRING, // name
+ TPrimitiveType::STRUCT, // profile
+ TPrimitiveType::ARRAY, // tags
+ TPrimitiveType::ARRAY, // friends
+ TPrimitiveType::ARRAY, // recent_activity
+ TPrimitiveType::MAP, // attributes
+ TPrimitiveType::MAP // complex_attributes
+ };
+
+ const TupleDescriptor* tuple_descriptor = create_tuple_descriptor(
+ &desc_tbl, obj_pool, t_desc_table, t_table_desc, table_column_names,
+ table_column_positions, table_column_types, access_configs);
+
+ // Execute test based on method choice
+ ColumnIdResult actual_result;
+ if (use_top_level_method) {
+ // actual_result = IcebergOrcReader::_create_column_ids_by_top_level_col_index(
+ // orc_type, tuple_descriptor);
+ } else {
+ actual_result = IcebergOrcReader::_create_column_ids(orc_type, tuple_descriptor);
+ }
+
+ if (!should_skip_assertion) {
+ EXPECT_EQ(actual_result.column_ids, expected_column_ids);
+ EXPECT_EQ(actual_result.filter_column_ids, expected_filter_column_ids);
+ }
+ }
+};
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_1) {
+ // Properties:
+
+ // Configure access paths for the profile column
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3", "9", "14", "15"},
+ {"3", "9", "14", "16"},
+ {"3", "10", "17"},
+ {"3", "11", "*", "23"}};
+ access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 7, 8, 9, 10, 11, 15, 16, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_2) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3"}};
+ access_config.predicate_paths = {{"3", "9", "14", "15"}, {"3", "10", "17"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 11, 12, 13, 14, 15, 16, 17, 18};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_3) {
+ // Properties:
+ // iceberg.schema: {"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":true,"type":"long"},{"id":2,"name":"name","required":true,"type":"string"},{"id":3,"name":"profile","required":true,"type":{"type":"struct","fields":[{"id":4,"name":"address","required":false,"type":{"type":"struct","fields":[{"id":7,"name":"street","required":false,"type":"string"},{"id":8,"name":"city","required":false,"type":"string"},{"id":9,"name":"coordinates","required":false,"type": [...]
+ // ORC 列ID分配为树型递增，根节点为0，子节点依次递增。
+ // 0: struct (table/root)
+ // 1: id (int64)
+ // 2: name (string)
+ // 3: profile (struct)
+ // 4: address (struct)
+ // 5: street (string)
+ // 6: city (string)
+ // 7: coordinates (struct)
+ // 8: lat (double)
+ // 9: lng (double)
+ // 10: contact (struct)
+ // 11: email (string)
+ // 12: phone (struct)
+ // 13: country_code (string)
+ // 14: number (string)
+ // 15: hobbies (list/array)
+ // 16: element (struct)
+ // 17: name (string)
+ // 18: level (int32)
+
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"3", "10"}, {"3", "9"}};
+ access_config.predicate_paths = {{"3", "9", "14"}, {"3", "10", "17"}};
+
+ // access_config.all_column_paths = {{"profile", "contact"}, {"profile", "address"}};
+ // access_config.predicate_paths = {{"profile", "address", "coordinates"},
+ // {"profile", "contact", "email"}};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {3, 4, 7, 8, 9, 10, 11};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_4) {
+ // 配置profile列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "profile";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {};
+ access_config.predicate_paths = {};
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {};
+
+ run_parquet_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method({access_config}, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_5) {
+ // 19: tags (array)
+ // 20: element (string)
+ // 21: friends (array)
+ // 22: element (struct)
+ // 23: user_id (bigint)
+ // 24: nickname (string)
+ // 25: friendship_level (int)
+ // 26: recent_activity (array)
+ // 27: element (struct)
+ // 28: action (string)
+ // 29: details (array)
+ // 30: element (struct)
+ // 31: key (string)
+ // 32: value (string)
+
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // 配置friends列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "friends";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"5", "*", "27"}, {"5", "*", "28"}};
+ access_config.predicate_paths = {
+ {"5", "*", "27"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // 配置recent_activity列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "recent_activity";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"6", "*", "30"}, {"6", "*", "31", "*", "34"}};
+ access_config.predicate_paths = {
+ {"6", "*", "30"},
+ };
+ access_configs.push_back(access_config);
+ }
+
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 21, 22, 24, 25, 26, 27, 28, 29, 30, 32};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {21, 22, 24, 26, 27, 28};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+}
+
+TEST_F(IcebergReaderCreateColumnIdsTest, test_create_column_ids_6) {
+ // 33: attributes (map)
+ // 34: key (string)
+ // 35: value (string)
+ // 36: complex_attributes (map)
+ // 37: key (string)
+ // 38: value (struct)
+ // 39: metadata (struct)
+ // 40: version (string)
+ // 41: created_time (timestamp)
+ // 42: last_updated (timestamp)
+ // 43: quality_score (double)
+ // 44: historical_scores (array)
+ // 45: element (struct)
+ // 46: period (string)
+ // 47: score (double)
+ // 48: components (array)
+ // 49: element (struct)
+ // 50: component_name (string)
+ // 51: weight (float)
+ // 52: sub_scores (map)
+ // 53: key (string)
+ // 54: value (double)
+ // 55: trends (struct)
+ // 56: direction (string)
+ // 57: magnitude (double)
+ // 58: confidence_interval (struct)
+ // 59: lower (double)
+ // 60: upper (double)
+ // 61: hierarchical_data (map)
+ // 62: key (string)
+ // 63: value (struct)
+ // 64: category (string)
+ // 65: sub_items (array)
+ // 66: element (struct)
+ // 67: item_id (bigint)
+ // 68: properties (map)
+ // 69: key (string)
+ // 70: value (string)
+ // 71: metrics (struct)
+ // 72: value (double)
+ // 73: unit (string)
+ // 74: summary (struct)
+ // 75: total_count (int)
+ // 76: average_value (double)
+ // 77: validation_rules (struct)
+ // 78: required (boolean)
+ // 79: constraints (array)
+ // 80: element (struct)
+ // 81: rule_type (string)
+ // 82: parameters (map)
+ // 83: key (string)
+ // 84: value (string)
+ // 85: error_message (string)
+ // 86: complex_constraint (struct)
+ // 87: logic_operator (string)
+ // 88: operands (array)
+ // 89: element (struct)
+ // 90: field_path (string)
+ // 91: operator (string)
+ // 92: comparison_value (string)
+ std::vector<ColumnAccessPathConfig> access_configs;
+
+ {
+ // 配置complex_attributes列的访问路径
+ ColumnAccessPathConfig access_config;
+ access_config.column_name = "complex_attributes";
+ access_config.use_name_paths = true;
+
+ access_config.all_column_paths = {{"8", "*", "39", "43"},
+ {"8", "*", "40", "*", "50", "*", "55"},
+ {"8", "*", "41", "VALUES"},
+ {"8", "VALUES", "42", "79", "*", "83", "KEYS"}};
+ access_config.predicate_paths = {{"8", "*", "39", "43"}};
+ access_configs.push_back(access_config);
+ }
+
+ {
+ // parquet values should access keys
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52, 53,
+ 54, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,
+ 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_parquet_test_with_method(access_configs, expected_column_ids,
+ expected_filter_column_ids);
+ }
+
+ {
+ // column_ids should contain all necessary column IDs (set automatically deduplicates)
+ // Expected IDs based on the schema: name(2), profile(3), address(4), coordinates(7), lat(8), lng(9), contact(10), email(11), hobbies(15), element(16), level(18)
+ std::set<uint64_t> expected_column_ids = {2, 36, 37, 38, 39, 40, 44, 45, 48, 49, 52,
+ 53, 54, 61, 63, 64, 65, 66, 67, 68, 69, 70,
+ 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 83};
+ // Expected IDs based on the schema: profile(3), address(4), coordinates(7), lat(8), contact(10), email(11)
+ std::set<uint64_t> expected_filter_column_ids = {36, 37, 38, 39, 40};
+
+ run_orc_test_with_method(access_configs, expected_column_ids, expected_filter_column_ids);
+ }
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
new file mode 100644
index 00000000000..12d6bb0299f
--- /dev/null
+++ b/be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
@@ -0,0 +1,739 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exec/format/table/iceberg_reader.h"
+
+#include <cctz/time_zone.h>
+#include <gen_cpp/Descriptors_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
+#include <gen_cpp/PlanNodes_types.h>
+#include <gen_cpp/Types_types.h>
+#include <gtest/gtest.h>
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/object_pool.h"
+#include "exec/olap_common.h"
+#include "io/fs/file_meta_cache.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_system.h"
+#include "io/fs/local_file_system.h"
+#include "runtime/descriptors.h"
+#include "runtime/runtime_state.h"
+#include "util/timezone_utils.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/core/block.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
+#include "vec/exec/format/parquet/vparquet_reader.h"
+
+namespace doris::vectorized {
+
+class IcebergReaderTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ cache = std::make_unique<doris::FileMetaCache>(1024);
+
+ // Setup timezone
+ doris::TimezoneUtils::find_cctz_time_zone(doris::TimezoneUtils::default_time_zone,
+ timezone_obj);
+ }
+
+ void TearDown() override { cache.reset(); }
+
+ // Helper function to create complex struct types for testing
+ void create_complex_struct_types(DataTypePtr& coordinates_struct_type,
+ DataTypePtr& address_struct_type,
+ DataTypePtr& phone_struct_type,
+ DataTypePtr& contact_struct_type,
+ DataTypePtr& hobby_element_struct_type,
+ DataTypePtr& hobbies_array_type,
+ DataTypePtr& profile_struct_type, DataTypePtr& name_type) {
+ // Create name column type (direct field)
+ name_type = make_nullable(std::make_shared<DataTypeString>());
+
+ // First create coordinates struct type
+ std::vector<DataTypePtr> coordinates_types = {
+ make_nullable(std::make_shared<DataTypeFloat64>()), // lat (field ID 10)
+ make_nullable(std::make_shared<DataTypeFloat64>()) // lng (field ID 11)
+ };
+ std::vector<std::string> coordinates_names = {"lat", "lng"};
+ coordinates_struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(coordinates_types, coordinates_names));
+
+ // Create address struct type (with street, city, coordinates)
+ std::vector<DataTypePtr> address_types = {
+ make_nullable(std::make_shared<DataTypeString>()), // street (field ID 7)
+ make_nullable(std::make_shared<DataTypeString>()), // city (field ID 8)
+ coordinates_struct_type // coordinates (field ID 9)
... 739 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org