You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2025/10/15 01:46:56 UTC
(doris) branch master updated: [fix](iceberg) support reading iceberg partition table without partition value stored in file (#56918)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b8688964673 [fix](iceberg) support reading iceberg partition table without partition value stored in file (#56918)
b8688964673 is described below
commit b86889646738be772449bbdba11cdd5a0ee51033
Author: Socrates <su...@icloud.com>
AuthorDate: Wed Oct 15 09:46:49 2025 +0800
[fix](iceberg) support reading iceberg partition table without partition value stored in file (#56918)
### What problem does this PR solve?
Fix querying iceberg partition table when partition value is not stored
in file.
---
be/src/common/config.cpp | 3 +++
be/src/common/config.h | 3 +++
be/src/vec/exec/scan/file_scanner.cpp | 11 +++++++++++
3 files changed, 17 insertions(+)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index dd789c7101a..5e7dec44248 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1494,6 +1494,9 @@ DEFINE_mBool(skip_loading_stale_rowset_meta, "false");
DEFINE_Bool(enable_file_logger, "true");
+// Enable partition column fallback when partition columns are missing from file
+DEFINE_Bool(enable_iceberg_partition_column_fallback, "true");
+
// The minimum row group size when exporting Parquet files. default 128MB
DEFINE_Int64(min_row_group_size, "134217728");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 84b189257f9..af81e4ff273 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1555,6 +1555,9 @@ DECLARE_mBool(skip_loading_stale_rowset_meta);
// Only works when starting BE with --console.
DECLARE_Bool(enable_file_logger);
+// Enable partition column fallback when partition columns are missing from file
+DECLARE_Bool(enable_iceberg_partition_column_fallback);
+
// The minimum row group size when exporting Parquet files.
DECLARE_Int64(min_row_group_size);
diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp
index 665e340363f..976dc7ade4e 100644
--- a/be/src/vec/exec/scan/file_scanner.cpp
+++ b/be/src/vec/exec/scan/file_scanner.cpp
@@ -1382,6 +1382,17 @@ Status FileScanner::_set_fill_or_truncate_columns(bool need_to_get_parsed_schema
_slot_lower_name_to_col_type.emplace(to_lower(col_name), col_type);
}
+ if (!_fill_partition_from_path && config::enable_iceberg_partition_column_fallback) {
+ // check if the cols of _partition_col_descs are in _missing_cols
+ // if so, set _fill_partition_from_path to true and remove the col from _missing_cols
+ for (const auto& [col_name, col_type] : _partition_col_descs) {
+ if (_missing_cols.contains(col_name)) {
+ _fill_partition_from_path = true;
+ _missing_cols.erase(col_name);
+ }
+ }
+ }
+
RETURN_IF_ERROR(_generate_missing_columns());
if (_fill_partition_from_path) {
RETURN_IF_ERROR(_cur_reader->set_fill_columns(_partition_col_descs, _missing_col_descs));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org