You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2025/12/25 04:03:43 UTC
(doris) branch tpc_preview4-external updated (139f505c3ac -> c7572648fb9)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
discard 139f505c3ac Optimize location for tpch1000 (#59218)
omit 7f2512b0e3a [Enhancement](parquet)update runtime filter when read next parquet row group.(#59053) (#59181)
omit d3852588a92 [opt](multi-catalog) Optimize file split size. (#59175)
omit 3d2a4d61cd1 Manifest cache for tpch1000 (#59178)
omit 362ad1e488e fix query error
omit 92db0ca2146 contine page
omit 53ac5c8d0b4 Revert "use bitshuffle to decoding"
omit c777b258f66 use bitshuffle to decoding
omit 3a2427d08b8 plain lz4 null page encode
omit b3aea4f57ff change the null map encode
add 128ecfbc330 [opt](pipeline) Avoid unnecessary column copying in the projection
add 22c6b543c36 add PASSTHROUGH in agg and join (#59302)
add 3cc5165d3d3 change the null map encode
add 69297e76312 plain lz4 null page encode
add ae0e410294b use bitshuffle to decoding
add 9a33e3ec66d Revert "use bitshuffle to decoding"
add 1432172e29b contine page
add 9a980895355 fix query error
add df02946fe26 support stream agg topn
new bbca9eacc3b Manifest cache for tpch1000 (#59178)
new bdaf8202784 [opt](multi-catalog) Optimize file split size. (#59175)
new 8147ffe4740 [Enhancement](parquet)update runtime filter when read next parquet row group.(#59053) (#59181)
new c7572648fb9 Optimize location for tpch1000 (#59218)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (139f505c3ac)
\
N -- N -- N refs/heads/tpc_preview4-external (c7572648fb9)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/pipeline/exec/hashjoin_probe_operator.h | 2 +
be/src/pipeline/exec/operator.cpp | 44 +--
be/src/pipeline/exec/operator.h | 2 +
.../exec/streaming_aggregation_operator.cpp | 301 ++++++++++++++++++---
.../pipeline/exec/streaming_aggregation_operator.h | 82 +++++-
.../glue/translator/PhysicalPlanTranslator.java | 3 +-
6 files changed, 364 insertions(+), 70 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/04: [Enhancement](parquet)update runtime filter when read next parquet row group.(#59053) (#59181)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8147ffe47402258e912ba54f4bb11605892be54f
Author: daidai <ch...@selectdb.com>
AuthorDate: Fri Dec 19 12:01:42 2025 +0800
[Enhancement](parquet)update runtime filter when read next parquet row group.(#59053) (#59181)
bp #59053
---
.../runtime_filter_consumer_helper.h | 2 +
.../exec/format/parquet/vparquet_group_reader.h | 7 +
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 38 ++++-
be/src/vec/exec/format/parquet/vparquet_reader.h | 16 ++
be/src/vec/exec/scan/file_scanner.cpp | 24 ++-
be/src/vec/exec/scan/file_scanner.h | 3 +-
be/src/vec/exec/scan/scanner.cpp | 1 +
.../scripts/create_preinstalled_scripts/run84.hql | 20 +++
.../runtime_filter_dim_small/dim_small.parquet | Bin 0 -> 4230 bytes
.../runtime_filter_fact_big/fact_big.parquet | Bin 0 -> 129338 bytes
.../hive/test_parquet_join_runtime_filter.groovy | 174 +++++++++++++++++++++
11 files changed, 272 insertions(+), 13 deletions(-)
diff --git a/be/src/runtime_filter/runtime_filter_consumer_helper.h b/be/src/runtime_filter/runtime_filter_consumer_helper.h
index 212df4338cb..36da3cd10c0 100644
--- a/be/src/runtime_filter/runtime_filter_consumer_helper.h
+++ b/be/src/runtime_filter/runtime_filter_consumer_helper.h
@@ -52,6 +52,8 @@ public:
// parent_operator_profile is owned by LocalState so update it is safe at here.
void collect_realtime_profile(RuntimeProfile* parent_operator_profile);
+ size_t runtime_filter_nums() const { return _runtime_filter_descs.size(); }
+
private:
// Append late-arrival runtime filters to the vconjunct_ctx.
Status _append_rf_into_conjuncts(RuntimeState* state,
diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
index 265a95f4470..f81d6607349 100644
--- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h
@@ -79,7 +79,14 @@ public:
// table name
struct LazyReadContext {
+ // all conjuncts: in sql, join runtime filter, topn runtime filter.
VExprContextSPtrs conjuncts;
+
+ // ParquetReader::set_fill_columns(xxx, xxx) will set these two members
+ std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
+ fill_partition_columns;
+ std::unordered_map<std::string, VExprContextSPtr> fill_missing_columns;
+
bool can_lazy_read = false;
// block->rows() returns the number of rows of the first column,
// so we should check and resize the first column
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index fb30e5d4a61..45cf3e2c5ed 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -383,11 +383,17 @@ bool ParquetReader::_type_matches(const VSlotRef* slot_ref) const {
!is_complex_type(table_col_type->get_primitive_type());
}
-Status ParquetReader::set_fill_columns(
- const std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>&
- partition_columns,
- const std::unordered_map<std::string, VExprContextSPtr>& missing_columns) {
- SCOPED_RAW_TIMER(&_statistics.parse_meta_time);
+Status ParquetReader::_update_lazy_read_ctx(const VExprContextSPtrs& new_conjuncts) {
+ RowGroupReader::LazyReadContext new_lazy_read_ctx;
+ new_lazy_read_ctx.conjuncts = new_conjuncts;
+ new_lazy_read_ctx.fill_partition_columns = std::move(_lazy_read_ctx.fill_partition_columns);
+ new_lazy_read_ctx.fill_missing_columns = std::move(_lazy_read_ctx.fill_missing_columns);
+ _lazy_read_ctx = std::move(new_lazy_read_ctx);
+
+ _top_runtime_vexprs.clear();
+ _push_down_predicates.clear();
+ _useless_predicates.clear();
+
// std::unordered_map<column_name, std::pair<col_id, slot_id>>
std::unordered_map<std::string, std::pair<uint32_t, int>> predicate_columns;
// visit_slot for lazy mat.
@@ -494,7 +500,7 @@ Status ParquetReader::set_fill_columns(
_lazy_read_ctx.all_predicate_col_ids.emplace_back(_row_id_column_iterator_pair.second);
}
- for (auto& kv : partition_columns) {
+ for (auto& kv : _lazy_read_ctx.fill_partition_columns) {
auto iter = predicate_columns.find(kv.first);
if (iter == predicate_columns.end()) {
_lazy_read_ctx.partition_columns.emplace(kv.first, kv.second);
@@ -504,7 +510,7 @@ Status ParquetReader::set_fill_columns(
}
}
- for (auto& kv : missing_columns) {
+ for (auto& kv : _lazy_read_ctx.fill_missing_columns) {
auto iter = predicate_columns.find(kv.first);
if (iter == predicate_columns.end()) {
_lazy_read_ctx.missing_columns.emplace(kv.first, kv.second);
@@ -536,6 +542,17 @@ Status ParquetReader::set_fill_columns(
}
}
+ return Status::OK();
+}
+
+Status ParquetReader::set_fill_columns(
+ const std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>&
+ partition_columns,
+ const std::unordered_map<std::string, VExprContextSPtr>& missing_columns) {
+ _lazy_read_ctx.fill_partition_columns = partition_columns;
+ _lazy_read_ctx.fill_missing_columns = missing_columns;
+ RETURN_IF_ERROR(_update_lazy_read_ctx(_lazy_read_ctx.conjuncts));
+
if (_filter_groups && (_total_groups == 0 || _t_metadata->num_rows == 0 || _range_size < 0)) {
return Status::EndOfFile("No row group to read");
}
@@ -673,6 +690,13 @@ Status ParquetReader::_next_row_group_reader() {
continue;
}
+ bool has_late_rf_cond = false;
+ VExprContextSPtrs new_push_down_conjuncts;
+ RETURN_IF_ERROR(_call_late_rf_func(&has_late_rf_cond, new_push_down_conjuncts));
+ if (has_late_rf_cond) {
+ RETURN_IF_ERROR(_update_lazy_read_ctx(new_push_down_conjuncts));
+ }
+
size_t before_predicate_size = _push_down_predicates.size();
_push_down_predicates.reserve(before_predicate_size + _top_runtime_vexprs.size());
for (const auto& vexpr : _top_runtime_vexprs) {
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h
index c3c73d98bea..e2ba5d82a70 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.h
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.h
@@ -160,6 +160,10 @@ public:
bool count_read_rows() override { return true; }
+ void set_update_late_rf_func(std::function<Status(bool*, VExprContextSPtrs&)>&& func) {
+ _call_late_rf_func = std::move(func);
+ }
+
protected:
void _collect_profile_before_close() override;
@@ -252,6 +256,9 @@ private:
bool _exists_in_file(const VSlotRef* slot) const override;
bool _type_matches(const VSlotRef*) const override;
+ // update lazy read context when runtime filter changed
+ Status _update_lazy_read_ctx(const VExprContextSPtrs& new_conjuncts);
+
RuntimeProfile* _profile = nullptr;
const TFileScanRangeParams& _scan_params;
const TFileRangeDesc& _scan_range;
@@ -337,6 +344,15 @@ private:
std::vector<std::unique_ptr<MutilColumnBlockPredicate>> _push_down_predicates;
std::vector<std::unique_ptr<ColumnPredicate>> _useless_predicates;
Arena _arena;
+
+ // when creating a new row group reader, call this function to get the latest runtime filter conjuncts.
+ // The default implementation does nothing, sets 'changed' to false, and returns OK.
+ // This is used when iceberg read position delete file ...
+ static Status default_late_rf_func(bool* changed, VExprContextSPtrs&) {
+ *changed = false;
+ return Status::OK();
+ }
+ std::function<Status(bool*, VExprContextSPtrs&)> _call_late_rf_func = default_late_rf_func;
};
#include "common/compile_check_end.h"
diff --git a/be/src/vec/exec/scan/file_scanner.cpp b/be/src/vec/exec/scan/file_scanner.cpp
index c7d10c89dc0..8629737a320 100644
--- a/be/src/vec/exec/scan/file_scanner.cpp
+++ b/be/src/vec/exec/scan/file_scanner.cpp
@@ -357,8 +357,11 @@ Status FileScanner::_process_conjuncts() {
return Status::OK();
}
-Status FileScanner::_process_late_arrival_conjuncts() {
+Status FileScanner::_process_late_arrival_conjuncts(bool* changed,
+ VExprContextSPtrs& new_push_down_conjuncts) {
+ *changed = false;
if (_push_down_conjuncts.size() < _conjuncts.size()) {
+ *changed = true;
_push_down_conjuncts.clear();
_push_down_conjuncts.resize(_conjuncts.size());
for (size_t i = 0; i != _conjuncts.size(); ++i) {
@@ -366,6 +369,7 @@ Status FileScanner::_process_late_arrival_conjuncts() {
}
RETURN_IF_ERROR(_process_conjuncts());
_discard_conjuncts();
+ new_push_down_conjuncts = _push_down_conjuncts;
}
if (_applied_rf_num == _total_rf_num) {
_local_state->scanner_profile()->add_info_string("ApplyAllRuntimeFilters", "True");
@@ -1045,9 +1049,17 @@ Status FileScanner::_get_next_reader() {
// ATTN: the push down agg type may be set back to NONE,
// see IcebergTableReader::init_row_filters for example.
parquet_reader->set_push_down_agg_type(_get_push_down_agg_type());
- if (push_down_predicates) {
- RETURN_IF_ERROR(_process_late_arrival_conjuncts());
- }
+
+ std::function<Status(bool*, VExprContextSPtrs&)> update_late_rf =
+ [&](bool* changed, VExprContextSPtrs& new_push_down_conjuncts) -> Status {
+ if (!_is_load) {
+ RETURN_IF_ERROR(try_append_late_arrival_runtime_filter());
+ RETURN_IF_ERROR(
+ _process_late_arrival_conjuncts(changed, new_push_down_conjuncts));
+ }
+ return Status::OK();
+ };
+ parquet_reader->set_update_late_rf_func(std::move(update_late_rf));
RETURN_IF_ERROR(_init_parquet_reader(std::move(parquet_reader), file_meta_cache_ptr));
need_to_get_parsed_schema = true;
@@ -1068,7 +1080,9 @@ Status FileScanner::_get_next_reader() {
orc_reader->set_push_down_agg_type(_get_push_down_agg_type());
if (push_down_predicates) {
- RETURN_IF_ERROR(_process_late_arrival_conjuncts());
+ bool changed = false;
+ VExprContextSPtrs new_push_down_conjuncts;
+ RETURN_IF_ERROR(_process_late_arrival_conjuncts(&changed, new_push_down_conjuncts));
}
RETURN_IF_ERROR(_init_orc_reader(std::move(orc_reader), file_meta_cache_ptr));
diff --git a/be/src/vec/exec/scan/file_scanner.h b/be/src/vec/exec/scan/file_scanner.h
index d26186eeef6..1cbe9c1bbcf 100644
--- a/be/src/vec/exec/scan/file_scanner.h
+++ b/be/src/vec/exec/scan/file_scanner.h
@@ -251,7 +251,8 @@ private:
void _init_runtime_filter_partition_prune_block();
Status _process_runtime_filters_partition_prune(bool& is_partition_pruned);
Status _process_conjuncts();
- Status _process_late_arrival_conjuncts();
+ Status _process_late_arrival_conjuncts(bool* changed,
+ VExprContextSPtrs& new_push_down_conjuncts);
void _get_slot_ids(VExpr* expr, std::vector<int>* slot_ids);
Status _generate_truncate_columns(bool need_to_get_parsed_schema);
Status _set_fill_or_truncate_columns(bool need_to_get_parsed_schema);
diff --git a/be/src/vec/exec/scan/scanner.cpp b/be/src/vec/exec/scan/scanner.cpp
index 5dced63feb6..2857738297f 100644
--- a/be/src/vec/exec/scan/scanner.cpp
+++ b/be/src/vec/exec/scan/scanner.cpp
@@ -41,6 +41,7 @@ Scanner::Scanner(RuntimeState* state, pipeline::ScanLocalStateBase* local_state,
_output_tuple_desc(_local_state->output_tuple_desc()),
_output_row_descriptor(_local_state->_parent->output_row_descriptor()),
_has_prepared(false) {
+ _total_rf_num = cast_set<int>(_local_state->_helper.runtime_filter_nums());
DorisMetrics::instance()->scanner_cnt->increment(1);
}
diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run84.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run84.hql
new file mode 100644
index 00000000000..4b4e7b6e549
--- /dev/null
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run84.hql
@@ -0,0 +1,20 @@
+use `default`;
+
+create table fact_big (
+ k INT,
+ c1 INT,
+ c2 BIGINT,
+ c3 DOUBLE,
+ c4 STRING
+)stored as parquet
+LOCATION '/user/doris/preinstalled_data/parquet_table/runtime_filter_fact_big';
+
+create table dim_small (
+ k INT,
+ c1 INT,
+ c2 BIGINT
+)stored as parquet
+LOCATION '/user/doris/preinstalled_data/parquet_table/runtime_filter_dim_small';
+
+msck repair table fact_big;
+msck repair table dim_small;
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_dim_small/dim_small.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_dim_small/dim_small.parquet
new file mode 100644
index 00000000000..e998f3c817a
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_dim_small/dim_small.parquet differ
diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_fact_big/fact_big.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_fact_big/fact_big.parquet
new file mode 100644
index 00000000000..b3ad736022e
Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_fact_big/fact_big.parquet differ
diff --git a/regression-test/suites/external_table_p0/hive/test_parquet_join_runtime_filter.groovy b/regression-test/suites/external_table_p0/hive/test_parquet_join_runtime_filter.groovy
new file mode 100644
index 00000000000..8c0b1516459
--- /dev/null
+++ b/regression-test/suites/external_table_p0/hive/test_parquet_join_runtime_filter.groovy
@@ -0,0 +1,174 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.json.JsonSlurper
+
+suite("test_parquet_join_runtime_filter", "p0,external,hive,external_docker,external_docker_hive") {
+
+ def getProfileList = {
+ def dst = 'http://' + context.config.feHttpAddress
+ def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
+ conn.setRequestMethod("GET")
+ def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+ (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ return conn.getInputStream().getText()
+ }
+
+ def getProfile = { id ->
+ def dst = 'http://' + context.config.feHttpAddress
+ def conn = new URL(dst + "/api/profile/text/?query_id=$id").openConnection()
+ conn.setRequestMethod("GET")
+ def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+ (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ return conn.getInputStream().getText()
+ }
+
+
+ def extractFilteredGroupsValue = { String profileText ->
+ def values = (profileText =~ /FilteredGroups:\s*(\d+)/).collect { it[1].toLong() }
+ return values.sort { a, b -> b <=> a }
+ }
+
+ def getProfileWithToken = { token ->
+ String profileId = ""
+ int attempts = 0
+ while (attempts < 10 && (profileId == null || profileId == "")) {
+ List profileData = new JsonSlurper().parseText(getProfileList()).data.rows
+ for (def profileItem in profileData) {
+ if (profileItem["Sql Statement"].toString().contains(token)) {
+ profileId = profileItem["Profile ID"].toString()
+ break
+ }
+ }
+ if (profileId == null || profileId == "") {
+ Thread.sleep(300)
+ }
+ attempts++
+ }
+ assertTrue(profileId != null && profileId != "")
+ Thread.sleep(800)
+ return getProfile(profileId).toString()
+ }
+ // session vars
+ sql "unset variable all;"
+ sql "set profile_level=2;"
+ sql "set enable_profile=true;"
+
+
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (!"true".equalsIgnoreCase(enabled)) {
+ return;
+ }
+ for (String hivePrefix : ["hive2"]) {
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String hmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+ String catalog_name = "test_parquet_join_runtime_filter"
+
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
+ create catalog if not exists ${catalog_name} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}'
+ );
+ """
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+
+ sql """ use `default` """
+
+
+ for (int wait_time : [0, 10, 100]) {
+ sql """ set runtime_filter_wait_time_ms = ${wait_time}; """
+
+ def f1 = {
+ def t1 = UUID.randomUUID().toString()
+ def sql_result = sql """
+ select *, "${t1}" from fact_big as a join dim_small as b on a.k = b.k where b.c1 = 5
+ """
+ def filter_result = extractFilteredGroupsValue(getProfileWithToken(t1));
+ logger.info("sql_result = ${sql_result}");
+ logger.info("filter_result = ${filter_result}");
+
+ assertTrue(filter_result.size() == 2)
+ assertTrue(filter_result[0] > 40)
+ }
+
+
+
+ def f2 = {
+ def t1 = UUID.randomUUID().toString()
+ def sql_result = sql """
+ select *, "${t1}" from fact_big as a join dim_small as b on a.k = b.k where b.c1 in (1,2)
+ """
+ def filter_result = extractFilteredGroupsValue(getProfileWithToken(t1));
+ logger.info("sql_result = ${sql_result}");
+ logger.info("filter_result = ${filter_result}");
+
+ assertTrue(filter_result.size() == 2)
+ assertTrue(filter_result[0] > 30)
+ }
+
+
+
+
+ def f3 = {
+ def t1 = UUID.randomUUID().toString()
+ def sql_result = sql """
+ select *, "${t1}" from fact_big as a join dim_small as b on a.k = b.k where b.c1 < 3
+ """
+ def filter_result = extractFilteredGroupsValue(getProfileWithToken(t1));
+ logger.info("sql_result = ${sql_result}");
+ logger.info("filter_result = ${filter_result}");
+
+ assertTrue(filter_result.size() == 2)
+ assertTrue(filter_result[0] > 30)
+ }
+
+
+
+ def f4 = {
+ def t1 = UUID.randomUUID().toString()
+ def sql_result = sql """
+ select *, "${t1}" from fact_big as a join dim_small as b on a.k = b.k where b.c2 >= 50
+ """
+ def filter_result = extractFilteredGroupsValue(getProfileWithToken(t1));
+ logger.info("sql_result = ${sql_result}");
+ logger.info("filter_result = ${filter_result}");
+
+ assertTrue(filter_result.size() == 2)
+ assertTrue(filter_result[0] > 40)
+ }
+
+
+ f1()
+ f2()
+ f3()
+ f4()
+ }
+
+ sql """drop catalog ${catalog_name};"""
+ }
+
+
+
+
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/04: [opt](multi-catalog) Optimize file split size. (#59175)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bdaf820278473d0e3a58faca1761822bfd376bb3
Author: Qi Chen <ch...@selectdb.com>
AuthorDate: Fri Dec 19 11:45:28 2025 +0800
[opt](multi-catalog) Optimize file split size. (#59175)
---
.../doris/datasource/FederationBackendPolicy.java | 12 +-
.../apache/doris/datasource/FileQueryScanNode.java | 19 +-
.../org/apache/doris/datasource/FileScanNode.java | 10 +-
.../org/apache/doris/datasource/FileSplitter.java | 230 ++++++++++++++++++---
.../apache/doris/datasource/SplitGenerator.java | 2 +-
.../doris/datasource/hive/source/HiveScanNode.java | 78 +++++--
.../datasource/iceberg/source/IcebergScanNode.java | 80 ++++++-
.../datasource/paimon/source/PaimonScanNode.java | 47 ++++-
.../doris/datasource/tvf/source/TVFScanNode.java | 31 ++-
.../java/org/apache/doris/qe/SessionVariable.java | 39 ++++
.../apache/doris/datasource/FileSplitterTest.java | 216 +++++++++++++++++++
.../paimon/source/PaimonScanNodeTest.java | 17 ++
.../doris/planner/FederationBackendPolicyTest.java | 4 +-
.../hive/test_hive_compress_type.groovy | 2 +-
14 files changed, 691 insertions(+), 96 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java
index 813d1892642..a8927d86a94 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FederationBackendPolicy.java
@@ -63,12 +63,16 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
public class FederationBackendPolicy {
private static final Logger LOG = LogManager.getLogger(FederationBackendPolicy.class);
+
+ private static final long FIXED_SHUFFLE_SEED = 123456789L;
+
protected final List<Backend> backends = Lists.newArrayList();
private final Map<String, List<Backend>> backendMap = Maps.newHashMap();
@@ -220,6 +224,7 @@ public class FederationBackendPolicy {
public Multimap<Backend, Split> computeScanRangeAssignment(List<Split> splits) throws UserException {
ListMultimap<Backend, Split> assignment = ArrayListMultimap.create();
+ Collections.shuffle(splits, new Random(FIXED_SHUFFLE_SEED));
List<Split> remainingSplits;
List<Backend> backends = new ArrayList<>();
@@ -228,8 +233,6 @@ public class FederationBackendPolicy {
}
ResettableRandomizedIterator<Backend> randomCandidates = new ResettableRandomizedIterator<>(backends);
- boolean splitsToBeRedistributed = false;
-
// optimizedLocalScheduling enables prioritized assignment of splits to local nodes when splits contain
// locality information
if (Config.split_assigner_optimized_local_scheduling) {
@@ -246,7 +249,6 @@ public class FederationBackendPolicy {
assignment.put(selectedBackend, split);
assignedWeightPerBackend.put(selectedBackend,
assignedWeightPerBackend.get(selectedBackend) + split.getSplitWeight().getRawValue());
- splitsToBeRedistributed = true;
continue;
}
}
@@ -276,7 +278,6 @@ public class FederationBackendPolicy {
case CONSISTENT_HASHING: {
candidateNodes = consistentHash.getNode(split,
Config.split_assigner_min_consistent_hash_candidate_num);
- splitsToBeRedistributed = true;
break;
}
default: {
@@ -302,7 +303,7 @@ public class FederationBackendPolicy {
assignedWeightPerBackend.get(selectedBackend) + split.getSplitWeight().getRawValue());
}
- if (enableSplitsRedistribution && splitsToBeRedistributed) {
+ if (enableSplitsRedistribution) {
equateDistribution(assignment);
}
return assignment;
@@ -499,3 +500,4 @@ public class FederationBackendPolicy {
}
}
}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index 93bec2d1849..3ae32170e4b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -94,6 +94,8 @@ public abstract class FileQueryScanNode extends FileScanNode {
protected TableScanParams scanParams;
+ protected FileSplitter fileSplitter;
+
/**
* External file scan node for Query hms table
* needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv
@@ -134,6 +136,8 @@ public abstract class FileQueryScanNode extends FileScanNode {
}
initBackendPolicy();
initSchemaParams();
+ fileSplitter = new FileSplitter(sessionVariable.maxInitialSplitSize, sessionVariable.maxSplitSize,
+ sessionVariable.maxInitialSplitNum);
}
// Init schema (Tuple/Slot) related params.
@@ -592,19 +596,4 @@ public abstract class FileQueryScanNode extends FileScanNode {
}
return this.scanParams;
}
-
- /**
- * The real file split size is determined by:
- * 1. If user specify the split size in session variable `file_split_size`, use user specified value.
- * 2. Otherwise, use the max value of DEFAULT_SPLIT_SIZE and block size.
- * @param blockSize, got from file system, eg, hdfs
- * @return the real file split size
- */
- protected long getRealFileSplitSize(long blockSize) {
- long realSplitSize = sessionVariable.getFileSplitSize();
- if (realSplitSize <= 0) {
- realSplitSize = Math.max(DEFAULT_SPLIT_SIZE, blockSize);
- }
- return realSplitSize;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
index c3e06999bba..a7aa0f607ac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java
@@ -62,9 +62,6 @@ import java.util.stream.Collectors;
* Base class for External File Scan, including external query and load.
*/
public abstract class FileScanNode extends ExternalScanNode {
-
- public static final long DEFAULT_SPLIT_SIZE = 64 * 1024 * 1024; // 64MB
-
// For explain
protected long totalFileSize = 0;
protected long totalPartitionNum = 0;
@@ -115,12 +112,7 @@ public abstract class FileScanNode extends ExternalScanNode {
}
output.append(prefix);
- boolean isBatch;
- try {
- isBatch = isBatchMode();
- } catch (UserException e) {
- throw new RuntimeException(e);
- }
+ boolean isBatch = isBatchMode();
if (isBatch) {
output.append("(approximate)");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java
index 33b2d70bfb1..5fe84441971 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileSplitter.java
@@ -22,13 +22,19 @@ import org.apache.doris.common.util.Util;
import org.apache.doris.spi.Split;
import org.apache.doris.thrift.TFileCompressType;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
public class FileSplitter {
private static final Logger LOG = LogManager.getLogger(FileSplitter.class);
@@ -40,18 +46,66 @@ public class FileSplitter {
return totalFileNum < parallelism * numBackends;
}
- public static List<Split> splitFile(
- LocationPath path,
- long fileSplitSize,
- BlockLocation[] blockLocations,
- long length,
- long modificationTime,
- boolean splittable,
- List<String> partitionValues,
- SplitCreator splitCreator)
- throws IOException {
+ private long maxInitialSplitSize;
+
+ private long maxSplitSize;
+
+ private int maxInitialSplitNum;
+ private final AtomicInteger remainingInitialSplitNum;
+
+ private long currentMaxSplitSize;
+
+ public long getMaxInitialSplitSize() {
+ return maxInitialSplitSize;
+ }
+
+ public void setMaxInitialSplitSize(long maxInitialSplitSize) {
+ this.maxInitialSplitSize = maxInitialSplitSize;
+ }
+
+ public long getMaxSplitSize() {
+ return maxSplitSize;
+ }
+
+ public void setMaxSplitSize(long maxSplitSize) {
+ this.maxSplitSize = maxSplitSize;
+ }
+
+ public int maxInitialSplitNum() {
+ return maxInitialSplitNum;
+ }
+
+ public void setMaxInitialSplits(int maxInitialSplitNum) {
+ this.maxInitialSplitNum = maxInitialSplitNum;
+ }
+
+ public long getRemainingInitialSplitNum() {
+ return remainingInitialSplitNum.get();
+ }
+
+ public FileSplitter(long maxInitialSplitSize, long maxSplitSize, int maxInitialSplitNum) {
+ this.maxInitialSplitSize = maxInitialSplitSize;
+ this.maxSplitSize = maxSplitSize;
+ this.maxInitialSplitNum = maxInitialSplitNum;
+ currentMaxSplitSize = maxInitialSplitSize;
+ remainingInitialSplitNum = new AtomicInteger(maxInitialSplitNum);
+ }
+
+ public List<Split> splitFile(
+ LocationPath path,
+ long specifiedFileSplitSize,
+ BlockLocation[] blockLocations,
+ long length,
+ long modificationTime,
+ boolean splittable,
+ List<String> partitionValues,
+ SplitCreator splitCreator)
+ throws IOException {
+ // Pass splitCreator.create() to set target file split size to calculate split weight.
+ long targetFileSplitSize = specifiedFileSplitSize > 0 ? specifiedFileSplitSize : maxSplitSize;
if (blockLocations == null) {
- blockLocations = new BlockLocation[0];
+ blockLocations = new BlockLocation[1];
+ blockLocations[0] = new BlockLocation(null, null, 0L, length);
}
List<Split> result = Lists.newArrayList();
TFileCompressType compressType = Util.inferFileCompressTypeByPath(path.getNormalizedLocation());
@@ -60,23 +114,83 @@ public class FileSplitter {
LOG.debug("Path {} is not splittable.", path);
}
String[] hosts = blockLocations.length == 0 ? null : blockLocations[0].getHosts();
- result.add(splitCreator.create(path, 0, length, length, fileSplitSize,
+ result.add(splitCreator.create(path, 0, length, length,
+ targetFileSplitSize,
modificationTime, hosts, partitionValues));
+ updateCurrentMaxSplitSize();
+ return result;
+ }
+
+ // if specified split size is not zero, split file by specified size
+ if (specifiedFileSplitSize > 0) {
+ long bytesRemaining;
+ for (bytesRemaining = length; (double) bytesRemaining / (double) specifiedFileSplitSize > 1.1D;
+ bytesRemaining -= specifiedFileSplitSize) {
+ int location = getBlockIndex(blockLocations, length - bytesRemaining);
+ String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
+ result.add(splitCreator.create(path, length - bytesRemaining, specifiedFileSplitSize,
+ length, specifiedFileSplitSize, modificationTime, hosts, partitionValues));
+ }
+ if (bytesRemaining != 0L) {
+ int location = getBlockIndex(blockLocations, length - bytesRemaining);
+ String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
+ result.add(splitCreator.create(path, length - bytesRemaining, bytesRemaining,
+ length, specifiedFileSplitSize, modificationTime, hosts, partitionValues));
+ }
return result;
}
- long bytesRemaining;
- for (bytesRemaining = length; (double) bytesRemaining / (double) fileSplitSize > 1.1D;
- bytesRemaining -= fileSplitSize) {
- int location = getBlockIndex(blockLocations, length - bytesRemaining);
- String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
- result.add(splitCreator.create(path, length - bytesRemaining, fileSplitSize,
- length, fileSplitSize, modificationTime, hosts, partitionValues));
+
+ // split file by block
+ long start = 0;
+ ImmutableList.Builder<InternalBlock> blockBuilder = ImmutableList.builder();
+ for (BlockLocation blockLocation : blockLocations) {
+ // clamp the block range
+ long blockStart = Math.max(start, blockLocation.getOffset());
+ long blockEnd = Math.min(start + length, blockLocation.getOffset() + blockLocation.getLength());
+ if (blockStart > blockEnd) {
+ // block is outside split range
+ continue;
+ }
+ if (blockStart == blockEnd && !(blockStart == start && blockEnd == start + length)) {
+ // skip zero-width block, except in the special circumstance:
+ // slice is empty, and the block covers the empty slice interval.
+ continue;
+ }
+ blockBuilder.add(new InternalBlock(blockStart, blockEnd, blockLocation.getHosts()));
+ }
+ List<InternalBlock> blocks = blockBuilder.build();
+ if (blocks.isEmpty()) {
+ result.add(splitCreator.create(path, 0, length, length,
+ targetFileSplitSize, modificationTime, null,
+ partitionValues));
+ updateCurrentMaxSplitSize();
+ return result;
}
- if (bytesRemaining != 0L) {
- int location = getBlockIndex(blockLocations, length - bytesRemaining);
- String[] hosts = location == -1 ? null : blockLocations[location].getHosts();
- result.add(splitCreator.create(path, length - bytesRemaining, bytesRemaining,
- length, fileSplitSize, modificationTime, hosts, partitionValues));
+
+ long splitStart = start;
+ int currentBlockIdx = 0;
+ while (splitStart < start + length) {
+ updateCurrentMaxSplitSize();
+ long splitBytes;
+ long remainingBlockBytes = blocks.get(currentBlockIdx).getEnd() - splitStart;
+ if (remainingBlockBytes <= currentMaxSplitSize) {
+ splitBytes = remainingBlockBytes;
+ } else if (currentMaxSplitSize * 2 >= remainingBlockBytes) {
+ // Second to last split in this block, generate two evenly sized splits
+ splitBytes = remainingBlockBytes / 2;
+ } else {
+ splitBytes = currentMaxSplitSize;
+ }
+ result.add(splitCreator.create(path, splitStart, splitBytes,
+ length, targetFileSplitSize, modificationTime, blocks.get(currentBlockIdx).getHosts(),
+ partitionValues));
+ splitStart += splitBytes;
+ if (splitStart == blocks.get(currentBlockIdx).getEnd()) {
+ currentBlockIdx++;
+ if (currentBlockIdx != blocks.size()) {
+ Verify.verify(splitStart == blocks.get(currentBlockIdx).getStart());
+ }
+ }
}
if (LOG.isDebugEnabled()) {
@@ -85,7 +199,19 @@ public class FileSplitter {
return result;
}
- private static int getBlockIndex(BlockLocation[] blkLocations, long offset) {
+ private void updateCurrentMaxSplitSize() {
+ currentMaxSplitSize = maxSplitSize;
+ int cur = remainingInitialSplitNum.get();
+ while (cur > 0) {
+ if (remainingInitialSplitNum.compareAndSet(cur, cur - 1)) {
+ currentMaxSplitSize = maxInitialSplitSize;
+ break;
+ }
+ cur = remainingInitialSplitNum.get();
+ }
+ }
+
+ private int getBlockIndex(BlockLocation[] blkLocations, long offset) {
if (blkLocations == null || blkLocations.length == 0) {
return -1;
}
@@ -100,5 +226,59 @@ public class FileSplitter {
throw new IllegalArgumentException(String.format("Offset %d is outside of file (0..%d)", offset, fileLength));
}
+ private static class InternalBlock {
+ private final long start;
+ private final long end;
+ private final String[] hosts;
+
+ public InternalBlock(long start, long end, String[] hosts) {
+ Preconditions.checkArgument(start <= end, "block end cannot be before block start");
+ this.start = start;
+ this.end = end;
+ this.hosts = hosts;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public long getEnd() {
+ return end;
+ }
+
+ public String[] getHosts() {
+ return hosts;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ InternalBlock that = (InternalBlock) o;
+ return start == that.start && end == that.end && Arrays.equals(hosts, that.hosts);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = Objects.hash(start, end);
+ result = 31 * result + Arrays.hashCode(hosts);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("InternalBlock{");
+ sb.append("start=").append(start);
+ sb.append(", end=").append(end);
+ sb.append(", hosts=").append(Arrays.toString(hosts));
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java
index 0b8a1022d5a..391552a5106 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java
@@ -40,7 +40,7 @@ public interface SplitGenerator {
/**
* Whether the producer(e.g. ScanNode) support batch mode.
*/
- default boolean isBatchMode() throws UserException {
+ default boolean isBatchMode() {
return false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index 744423f622c..5bcf2f5546a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -186,7 +186,7 @@ public class HiveScanNode extends FileQueryScanNode {
.getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog());
String bindBrokerName = hmsTable.getCatalog().bindBrokerName();
List<Split> allFiles = Lists.newArrayList();
- getFileSplitByPartitions(cache, prunedPartitions, allFiles, bindBrokerName, numBackends);
+ getFileSplitByPartitions(cache, prunedPartitions, allFiles, bindBrokerName, numBackends, false);
if (ConnectContext.get().getExecutor() != null) {
ConnectContext.get().getExecutor().getSummaryProfile().setGetPartitionFilesFinishTime();
}
@@ -226,7 +226,8 @@ public class HiveScanNode extends FileQueryScanNode {
try {
List<Split> allFiles = Lists.newArrayList();
getFileSplitByPartitions(
- cache, Collections.singletonList(partition), allFiles, bindBrokerName, numBackends);
+ cache, Collections.singletonList(partition), allFiles, bindBrokerName,
+ numBackends, true);
if (allFiles.size() > numSplitsPerPartition.get()) {
numSplitsPerPartition.set(allFiles.size());
}
@@ -277,7 +278,8 @@ public class HiveScanNode extends FileQueryScanNode {
}
private void getFileSplitByPartitions(HiveMetaStoreCache cache, List<HivePartition> partitions,
- List<Split> allFiles, String bindBrokerName, int numBackends) throws IOException, UserException {
+ List<Split> allFiles, String bindBrokerName, int numBackends,
+ boolean isBatchMode) throws IOException, UserException {
List<FileCacheValue> fileCaches;
if (hiveTransaction != null) {
try {
@@ -293,9 +295,11 @@ public class HiveScanNode extends FileQueryScanNode {
fileCaches = cache.getFilesByPartitions(partitions, withCache, partitions.size() > 1,
directoryLister, hmsTable);
}
+
+ long targetFileSplitSize = determineTargetFileSplitSize(fileCaches, isBatchMode);
if (tableSample != null) {
List<HiveMetaStoreCache.HiveFileStatus> hiveFileStatuses = selectFiles(fileCaches);
- splitAllFiles(allFiles, hiveFileStatuses);
+ splitAllFiles(allFiles, hiveFileStatuses, targetFileSplitSize);
return;
}
@@ -319,27 +323,67 @@ public class HiveScanNode extends FileQueryScanNode {
int parallelNum = sessionVariable.getParallelExecInstanceNum();
needSplit = FileSplitter.needSplitForCountPushdown(parallelNum, numBackends, totalFileNum);
}
+
for (HiveMetaStoreCache.FileCacheValue fileCacheValue : fileCaches) {
- if (fileCacheValue.getFiles() != null) {
- boolean isSplittable = fileCacheValue.isSplittable();
- for (HiveMetaStoreCache.HiveFileStatus status : fileCacheValue.getFiles()) {
- allFiles.addAll(FileSplitter.splitFile(status.getPath(),
- // set block size to Long.MAX_VALUE to avoid splitting the file.
- getRealFileSplitSize(needSplit ? status.getBlockSize() : Long.MAX_VALUE),
- status.getBlockLocations(), status.getLength(), status.getModificationTime(),
- isSplittable, fileCacheValue.getPartitionValues(),
- new HiveSplitCreator(fileCacheValue.getAcidInfo())));
+ if (fileCacheValue.getFiles() == null) {
+ continue;
+ }
+ boolean isSplittable = fileCacheValue.isSplittable();
+
+ for (HiveMetaStoreCache.HiveFileStatus status : fileCacheValue.getFiles()) {
+ allFiles.addAll(fileSplitter.splitFile(
+ status.getPath(),
+ targetFileSplitSize,
+ status.getBlockLocations(),
+ status.getLength(),
+ status.getModificationTime(),
+ isSplittable && needSplit,
+ fileCacheValue.getPartitionValues(),
+ new HiveSplitCreator(fileCacheValue.getAcidInfo())));
+ }
+ }
+ }
+
+ private long determineTargetFileSplitSize(List<FileCacheValue> fileCaches,
+ boolean isBatchMode) {
+ if (sessionVariable.getFileSplitSize() > 0) {
+ return sessionVariable.getFileSplitSize();
+ }
+ /** Hive batch split mode will return 0. and <code>FileSplitter</code>
+ * will determine file split size.
+ */
+ if (isBatchMode) {
+ return 0;
+ }
+ long result = sessionVariable.getMaxInitialSplitSize();
+ long totalFileSize = 0;
+ for (HiveMetaStoreCache.FileCacheValue fileCacheValue : fileCaches) {
+ if (fileCacheValue.getFiles() == null) {
+ continue;
+ }
+ for (HiveMetaStoreCache.HiveFileStatus status : fileCacheValue.getFiles()) {
+ totalFileSize += status.getLength();
+ if (totalFileSize >= sessionVariable.getMaxSplitSize() * sessionVariable.getMaxInitialSplitNum()) {
+ result = sessionVariable.getMaxSplitSize();
+ break;
}
}
}
+ return result;
}
private void splitAllFiles(List<Split> allFiles,
- List<HiveMetaStoreCache.HiveFileStatus> hiveFileStatuses) throws IOException {
+ List<HiveMetaStoreCache.HiveFileStatus> hiveFileStatuses,
+ long realFileSplitSize) throws IOException {
for (HiveMetaStoreCache.HiveFileStatus status : hiveFileStatuses) {
- allFiles.addAll(FileSplitter.splitFile(status.getPath(), getRealFileSplitSize(status.getBlockSize()),
- status.getBlockLocations(), status.getLength(), status.getModificationTime(),
- status.isSplittable(), status.getPartitionValues(),
+ allFiles.addAll(fileSplitter.splitFile(
+ status.getPath(),
+ realFileSplitSize,
+ status.getBlockLocations(),
+ status.getLength(),
+ status.getModificationTime(),
+ status.isSplittable(),
+ status.getPartitionValues(),
new HiveSplitCreator(status.getAcidInfo())));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index 0ffe86edb31..133ac067644 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -85,6 +85,7 @@ import org.apache.iceberg.expressions.ResidualEvaluator;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.CloseableIterator;
import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.util.ScanTaskUtil;
import org.apache.iceberg.util.TableScanUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -119,7 +120,7 @@ public class IcebergScanNode extends FileQueryScanNode {
private boolean tableLevelPushDownCount = false;
private long countFromSnapshot;
private static final long COUNT_WITH_PARALLEL_SPLITS = 10000;
- private long targetSplitSize;
+ private long targetSplitSize = 0;
// This is used to avoid repeatedly calculating partition info map for the same partition data.
private Map<PartitionData, Map<String, String>> partitionMapInfos;
private boolean isPartitionedTable;
@@ -131,6 +132,8 @@ public class IcebergScanNode extends FileQueryScanNode {
private Map<StorageProperties.Type, StorageProperties> storagePropertiesMap;
private Map<String, String> backendStorageProperties;
+ private Boolean isBatchMode = null;
+
// for test
@VisibleForTesting
public IcebergScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable sv) {
@@ -171,7 +174,6 @@ public class IcebergScanNode extends FileQueryScanNode {
@Override
protected void doInitialize() throws UserException {
icebergTable = source.getIcebergTable();
- targetSplitSize = getRealFileSplitSize(0);
partitionMapInfos = new HashMap<>();
isPartitionedTable = icebergTable.spec().isPartitioned();
formatVersion = ((BaseTable) icebergTable).operations().current().formatVersion();
@@ -375,18 +377,57 @@ public class IcebergScanNode extends FileQueryScanNode {
private CloseableIterable<FileScanTask> planFileScanTask(TableScan scan) {
if (!Config.iceberg_manifest_cache_enable) {
- long targetSplitSize = getRealFileSplitSize(0);
- return TableScanUtil.splitFiles(scan.planFiles(), targetSplitSize);
+ return splitFiles(scan);
}
try {
return planFileScanTaskWithManifestCache(scan);
} catch (Exception e) {
LOG.warn("Plan with manifest cache failed, fallback to original scan: {}", e.getMessage());
- long targetSplitSize = getRealFileSplitSize(0);
- return TableScanUtil.splitFiles(scan.planFiles(), targetSplitSize);
+ return splitFiles(scan);
}
}
+ private CloseableIterable<FileScanTask> splitFiles(TableScan scan) {
+ if (sessionVariable.getFileSplitSize() > 0) {
+ return TableScanUtil.splitFiles(scan.planFiles(),
+ sessionVariable.getFileSplitSize());
+ }
+ if (isBatchMode()) {
+ // Currently iceberg batch split mode will use max split size.
+ // TODO: dynamic split size in batch split mode need to customize iceberg splitter.
+ return TableScanUtil.splitFiles(scan.planFiles(), sessionVariable.getMaxSplitSize());
+ }
+
+ // Non Batch Mode
+ // Materialize planFiles() into a list to avoid iterating the CloseableIterable twice.
+ // RISK: It will cost memory if the table is large.
+ List<FileScanTask> fileScanTaskList = new ArrayList<>();
+ try (CloseableIterable<FileScanTask> scanTasksIter = scan.planFiles()) {
+ for (FileScanTask task : scanTasksIter) {
+ fileScanTaskList.add(task);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to materialize file scan tasks", e);
+ }
+
+ targetSplitSize = determineTargetFileSplitSize(fileScanTaskList);
+ return TableScanUtil.splitFiles(CloseableIterable.withNoopClose(fileScanTaskList), targetSplitSize);
+ }
+
+ private long determineTargetFileSplitSize(Iterable<FileScanTask> tasks) {
+ long result = sessionVariable.getMaxInitialSplitSize();
+ long accumulatedTotalFileSize = 0;
+ for (FileScanTask task : tasks) {
+ accumulatedTotalFileSize += ScanTaskUtil.contentSizeInBytes(task.file());
+ if (accumulatedTotalFileSize
+ >= sessionVariable.getMaxSplitSize() * sessionVariable.getMaxInitialSplitNum()) {
+ result = sessionVariable.getMaxSplitSize();
+ break;
+ }
+ }
+ return result;
+ }
+
private CloseableIterable<FileScanTask> planFileScanTaskWithManifestCache(TableScan scan) throws IOException {
// Get the snapshot from the scan; return empty if no snapshot exists
Snapshot snapshot = scan.snapshot();
@@ -502,7 +543,7 @@ public class IcebergScanNode extends FileQueryScanNode {
}
// Split tasks into smaller chunks based on target split size for parallel processing
- long targetSplitSize = getRealFileSplitSize(0);
+ targetSplitSize = determineTargetFileSplitSize(tasks);
return TableScanUtil.splitFiles(CloseableIterable.withNoopClose(tasks), targetSplitSize);
}
@@ -592,21 +633,36 @@ public class IcebergScanNode extends FileQueryScanNode {
}
@Override
- public boolean isBatchMode() throws UserException {
+ public boolean isBatchMode() {
+ Boolean cached = isBatchMode;
+ if (cached != null) {
+ return cached;
+ }
TPushAggOp aggOp = getPushDownAggNoGroupingOp();
if (aggOp.equals(TPushAggOp.COUNT)) {
- countFromSnapshot = getCountFromSnapshot();
+ try {
+ countFromSnapshot = getCountFromSnapshot();
+ } catch (UserException e) {
+ throw new RuntimeException(e);
+ }
if (countFromSnapshot >= 0) {
tableLevelPushDownCount = true;
+ isBatchMode = false;
return false;
}
}
- if (createTableScan().snapshot() == null) {
- return false;
+ try {
+ if (createTableScan().snapshot() == null) {
+ isBatchMode = false;
+ return false;
+ }
+ } catch (UserException e) {
+ throw new RuntimeException(e);
}
if (!sessionVariable.getEnableExternalTableBatchMode()) {
+ isBatchMode = false;
return false;
}
@@ -622,10 +678,12 @@ public class IcebergScanNode extends FileQueryScanNode {
ManifestFile next = matchingManifest.next();
cnt += next.addedFilesCount() + next.existingFilesCount();
if (cnt >= sessionVariable.getNumFilesInBatchMode()) {
+ isBatchMode = true;
return true;
}
}
}
+ isBatchMode = false;
return false;
});
} catch (Exception e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index 402bf3d0ef6..1671ce0f17a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -28,7 +28,6 @@ import org.apache.doris.common.util.FileFormatUtils;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.datasource.ExternalUtil;
import org.apache.doris.datasource.FileQueryScanNode;
-import org.apache.doris.datasource.FileSplitter;
import org.apache.doris.datasource.credentials.CredentialUtils;
import org.apache.doris.datasource.credentials.VendedCredentialsFactory;
import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
@@ -293,7 +292,8 @@ public class PaimonScanNode extends FileQueryScanNode {
// And for counting the number of selected partitions for this paimon table.
Map<BinaryRow, Map<String, String>> partitionInfoMaps = new HashMap<>();
// if applyCountPushdown is true, we can't split the DataSplit
- long realFileSplitSize = getRealFileSplitSize(applyCountPushdown ? Long.MAX_VALUE : 0);
+ boolean hasDeterminedTargetFileSplitSize = false;
+ long targetFileSplitSize = 0;
for (DataSplit dataSplit : dataSplits) {
SplitStat splitStat = new SplitStat();
splitStat.setRowCount(dataSplit.rowCount());
@@ -325,6 +325,10 @@ public class PaimonScanNode extends FileQueryScanNode {
if (ignoreSplitType == SessionVariable.IgnoreSplitType.IGNORE_NATIVE) {
continue;
}
+ if (!hasDeterminedTargetFileSplitSize) {
+ targetFileSplitSize = determineTargetFileSplitSize(dataSplits, isBatchMode());
+ hasDeterminedTargetFileSplitSize = true;
+ }
splitStat.setType(SplitReadType.NATIVE);
splitStat.setRawFileConvertable(true);
List<RawFile> rawFiles = optRawFiles.get();
@@ -332,13 +336,13 @@ public class PaimonScanNode extends FileQueryScanNode {
RawFile file = rawFiles.get(i);
LocationPath locationPath = LocationPath.of(file.path(), storagePropertiesMap);
try {
- List<Split> dorisSplits = FileSplitter.splitFile(
+ List<Split> dorisSplits = fileSplitter.splitFile(
locationPath,
- realFileSplitSize,
+ targetFileSplitSize,
null,
file.length(),
-1,
- true,
+ !applyCountPushdown,
null,
PaimonSplit.PaimonSplitCreator.DEFAULT);
for (Split dorisSplit : dorisSplits) {
@@ -383,12 +387,43 @@ public class PaimonScanNode extends FileQueryScanNode {
// We need to set the target size for all splits so that we can calculate the
// proportion of each split later.
- splits.forEach(s -> s.setTargetSplitSize(realFileSplitSize));
+ splits.forEach(s -> s.setTargetSplitSize(sessionVariable.getFileSplitSize() > 0
+ ? sessionVariable.getFileSplitSize() : sessionVariable.getMaxSplitSize()));
this.selectedPartitionNum = partitionInfoMaps.size();
return splits;
}
+ private long determineTargetFileSplitSize(List<DataSplit> dataSplits,
+ boolean isBatchMode) {
+ if (sessionVariable.getFileSplitSize() > 0) {
+ return sessionVariable.getFileSplitSize();
+ }
+ /** Paimon batch split mode will return 0. and <code>FileSplitter</code>
+ * will determine file split size.
+ */
+ if (isBatchMode) {
+ return 0;
+ }
+ long result = sessionVariable.getMaxInitialSplitSize();
+ long totalFileSize = 0;
+ for (DataSplit dataSplit : dataSplits) {
+ Optional<List<RawFile>> rawFiles = dataSplit.convertToRawFiles();
+ if (!supportNativeReader(rawFiles)) {
+ continue;
+ }
+ for (RawFile rawFile : rawFiles.get()) {
+ totalFileSize += rawFile.fileSize();
+ if (totalFileSize
+ >= sessionVariable.getMaxSplitSize() * sessionVariable.getMaxInitialSplitNum()) {
+ result = sessionVariable.getMaxSplitSize();
+ break;
+ }
+ }
+ }
+ return result;
+ }
+
@VisibleForTesting
public Map<String, String> getIncrReadParams() throws UserException {
Map<String, String> paimonScanParams = new HashMap<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
index e7567559762..c3b0e3e8b6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java
@@ -146,12 +146,18 @@ public class TVFScanNode extends FileQueryScanNode {
needSplit = FileSplitter.needSplitForCountPushdown(parallelNum, numBackends, totalFileNum);
}
+ long targetFileSplitSize = determineTargetFileSplitSize(fileStatuses);
+
for (TBrokerFileStatus fileStatus : fileStatuses) {
try {
- splits.addAll(FileSplitter.splitFile(LocationPath.of(fileStatus.getPath()),
- getRealFileSplitSize(needSplit ? fileStatus.getBlockSize() : Long.MAX_VALUE),
- null, fileStatus.getSize(),
- fileStatus.getModificationTime(), fileStatus.isSplitable, null,
+ splits.addAll(fileSplitter.splitFile(
+ LocationPath.of(fileStatus.getPath()),
+ targetFileSplitSize,
+ null,
+ fileStatus.getSize(),
+ fileStatus.getModificationTime(),
+ fileStatus.isSplitable && needSplit,
+ null,
FileSplitCreator.DEFAULT));
} catch (IOException e) {
LOG.warn("get file split failed for TVF: {}", fileStatus.getPath(), e);
@@ -161,6 +167,23 @@ public class TVFScanNode extends FileQueryScanNode {
return splits;
}
+ private long determineTargetFileSplitSize(List<TBrokerFileStatus> fileStatuses) {
+ if (sessionVariable.getFileSplitSize() > 0) {
+ return sessionVariable.getFileSplitSize();
+ }
+ long result = sessionVariable.getMaxInitialSplitSize();
+ long totalFileSize = 0;
+ for (TBrokerFileStatus fileStatus : fileStatuses) {
+ totalFileSize += fileStatus.getSize();
+ if (totalFileSize
+ >= sessionVariable.getMaxSplitSize() * sessionVariable.getMaxInitialSplitNum()) {
+ result = sessionVariable.getMaxSplitSize();
+ break;
+ }
+ }
+ return result;
+ }
+
@Override
protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
if (split instanceof FileSplit) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 4fddfd0332d..14a25a992b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -513,6 +513,12 @@ public class SessionVariable implements Serializable, Writable {
// Split size for ExternalFileScanNode. Default value 0 means use the block size of HDFS/S3.
public static final String FILE_SPLIT_SIZE = "file_split_size";
+ public static final String MAX_INITIAL_FILE_SPLIT_SIZE = "max_initial_file_split_size";
+
+ public static final String MAX_FILE_SPLIT_SIZE = "max_file_split_size";
+
+ public static final String MAX_INITIAL_FILE_SPLIT_NUM = "max_initial_file_split_num";
+
// Target file size in bytes for Iceberg write operations
public static final String ICEBERG_WRITE_TARGET_FILE_SIZE_BYTES = "iceberg_write_target_file_size_bytes";
@@ -2161,6 +2167,15 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = FILE_SPLIT_SIZE, needForward = true)
public long fileSplitSize = 0;
+ @VariableMgr.VarAttr(name = MAX_INITIAL_FILE_SPLIT_SIZE, needForward = true)
+ public long maxInitialSplitSize = 32L * 1024L * 1024L;
+
+ @VariableMgr.VarAttr(name = MAX_FILE_SPLIT_SIZE, needForward = true)
+ public long maxSplitSize = 64L * 1024L * 1024L;
+
+ @VariableMgr.VarAttr(name = MAX_INITIAL_FILE_SPLIT_NUM, needForward = true)
+ public int maxInitialSplitNum = 200;
+
// Target file size for Iceberg write operations
// Default 0 means use config::iceberg_sink_max_file_size
@VariableMgr.VarAttr(name = ICEBERG_WRITE_TARGET_FILE_SIZE_BYTES, needForward = true)
@@ -4181,6 +4196,30 @@ public class SessionVariable implements Serializable, Writable {
this.fileSplitSize = fileSplitSize;
}
+ public long getMaxInitialSplitSize() {
+ return maxInitialSplitSize;
+ }
+
+ public void setMaxInitialSplitSize(long maxInitialSplitSize) {
+ this.maxInitialSplitSize = maxInitialSplitSize;
+ }
+
+ public long getMaxSplitSize() {
+ return maxSplitSize;
+ }
+
+ public void setMaxSplitSize(long maxSplitSize) {
+ this.maxSplitSize = maxSplitSize;
+ }
+
+ public int getMaxInitialSplitNum() {
+ return maxInitialSplitNum;
+ }
+
+ public void setMaxInitialSplitNum(int maxInitialSplitNum) {
+ this.maxInitialSplitNum = maxInitialSplitNum;
+ }
+
public long getIcebergWriteTargetFileSizeBytes() {
return icebergWriteTargetFileSizeBytes;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileSplitterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileSplitterTest.java
new file mode 100644
index 00000000000..a455923da4d
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileSplitterTest.java
@@ -0,0 +1,216 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource;
+
+import org.apache.doris.common.util.LocationPath;
+import org.apache.doris.spi.Split;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+public class FileSplitterTest {
+
+ private static final long MB = 1024L * 1024L;
+
+ private static final int DEFAULT_INITIAL_SPLITS = 200;
+
+ @Test
+ public void testNonSplittableCompressedFileProducesSingleSplit() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/file.gz");
+ BlockLocation[] locations = new BlockLocation[]{new BlockLocation(null, new String[]{"h1"}, 0L, 10 * MB)};
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, DEFAULT_INITIAL_SPLITS);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ 10 * MB,
+ 0L,
+ true,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+ Assert.assertEquals(1, splits.size());
+ Split s = splits.get(0);
+ Assert.assertEquals(10 * MB, ((org.apache.doris.datasource.FileSplit) s).getLength());
+ // host should be preserved
+ Assert.assertArrayEquals(new String[]{"h1"}, ((org.apache.doris.datasource.FileSplit) s).getHosts());
+ Assert.assertEquals(DEFAULT_INITIAL_SPLITS - 1, fileSplitter.getRemainingInitialSplitNum());
+ }
+
+ @Test
+ public void testEmptyBlockLocationsProducesSingleSplitAndNullHosts() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/file");
+ BlockLocation[] locations = new BlockLocation[0];
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, DEFAULT_INITIAL_SPLITS);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ 5 * MB,
+ 0L,
+ true,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+ Assert.assertEquals(1, splits.size());
+ org.apache.doris.datasource.FileSplit s = (org.apache.doris.datasource.FileSplit) splits.get(0);
+ Assert.assertEquals(5 * MB, s.getLength());
+ // hosts should be empty array when passing null
+ Assert.assertNotNull(s.getHosts());
+ Assert.assertEquals(0, s.getHosts().length);
+ }
+
+ @Test
+ public void testSplittableSingleBigBlockProducesExpectedSplitsWithInitialSmallChunks() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/bigfile");
+ long length = 200 * MB;
+ BlockLocation[] locations = new BlockLocation[]{new BlockLocation(null, new String[]{"h1"}, 0L, length)};
+ // set maxInitialSplits to 2 to force the first two splits to be small.
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, 2);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ length,
+ 0L,
+ true,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+
+ // expect splits sizes: 32MB, 32MB, 64MB, 36MB, 36MB -> sum is 200MB
+ long[] expected = new long[]{32 * MB, 32 * MB, 64 * MB, 36 * MB, 36 * MB};
+ Assert.assertEquals(expected.length, splits.size());
+ long sum = 0L;
+ for (int i = 0; i < expected.length; i++) {
+ org.apache.doris.datasource.FileSplit s = (org.apache.doris.datasource.FileSplit) splits.get(i);
+ Assert.assertEquals(expected[i], s.getLength());
+ sum += s.getLength();
+ // ensure host preserved
+ Assert.assertArrayEquals(new String[]{"h1"}, s.getHosts());
+ }
+ Assert.assertEquals(length, sum);
+ // ensure the initial small-split counter is consumed for the two initial small splits
+ Assert.assertEquals(0, fileSplitter.getRemainingInitialSplitNum());
+ }
+
+ @Test
+ public void testMultiBlockSplitsAndHostPreservation() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/twoblocks");
+ long len = 96 * MB;
+ BlockLocation[] locations = new BlockLocation[]{
+ new BlockLocation(null, new String[]{"h1"}, 0L, 48 * MB),
+ new BlockLocation(null, new String[]{"h2"}, 48 * MB, 48 * MB)
+ };
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, 0);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ len,
+ 0L,
+ true,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+ Assert.assertEquals(2, splits.size());
+ FileSplit s0 = (FileSplit) splits.get(0);
+ FileSplit s1 = (FileSplit) splits.get(1);
+ Assert.assertEquals(48 * MB, s0.getLength());
+ Assert.assertEquals(48 * MB, s1.getLength());
+ Assert.assertArrayEquals(new String[]{"h1"}, s0.getHosts());
+ Assert.assertArrayEquals(new String[]{"h2"}, s1.getHosts());
+ }
+
+ @Test
+ public void testZeroLengthBlockIsSkipped() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/zeroblock");
+ long length = 10 * MB;
+ BlockLocation[] locations = new BlockLocation[]{
+ new BlockLocation(null, new String[]{"h1"}, 0L, 0L),
+ new BlockLocation(null, new String[]{"h1"}, 0L, 10 * MB)
+ };
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, DEFAULT_INITIAL_SPLITS);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ length,
+ 0L,
+ true,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+ Assert.assertEquals(1, splits.size());
+ FileSplit s = (FileSplit) splits.get(0);
+ Assert.assertEquals(10 * MB, s.getLength());
+ Assert.assertArrayEquals(new String[]{"h1"}, s.getHosts());
+ }
+
+ @Test
+ public void testNonSplittableFlagDecrementsCounter() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/file.gz");
+ BlockLocation[] locations = new BlockLocation[]{new BlockLocation(null, new String[]{"h1"}, 0L, 10 * MB)};
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, 2);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ 10 * MB,
+ 0L,
+ false,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+ Assert.assertEquals(1, splits.size());
+ }
+
+ @Test
+ public void testNullRemainingInitialSplitIsAllowed() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/somefile");
+ BlockLocation[] locations = new BlockLocation[]{new BlockLocation(null, new String[]{"h1"}, 0L, 10 * MB)};
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, DEFAULT_INITIAL_SPLITS);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ 10 * MB,
+ 0L,
+ true,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+ Assert.assertEquals(1, splits.size());
+ }
+
+ @Test
+ public void testSmallFileNoSplit() throws Exception {
+ LocationPath loc = LocationPath.of("hdfs://example.com/path/small");
+ BlockLocation[] locations = new BlockLocation[]{new BlockLocation(null, new String[]{"h1"}, 0L, 2 * MB)};
+ FileSplitter fileSplitter = new FileSplitter(32 * MB, 64 * MB, DEFAULT_INITIAL_SPLITS);
+ List<Split> splits = fileSplitter.splitFile(
+ loc,
+ 0L,
+ locations,
+ 2 * MB,
+ 0L,
+ true,
+ Collections.emptyList(),
+ FileSplit.FileSplitCreator.DEFAULT);
+ Assert.assertEquals(1, splits.size());
+ FileSplit s = (FileSplit) splits.get(0);
+ Assert.assertEquals(2 * MB, s.getLength());
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
index 93afa390530..692a0db12ca 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
@@ -21,6 +21,8 @@ import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.FileQueryScanNode;
+import org.apache.doris.datasource.FileSplitter;
import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog;
import org.apache.doris.planner.PlanNodeId;
import org.apache.doris.qe.SessionVariable;
@@ -92,11 +94,26 @@ public class PaimonScanNodeTest {
}
}).when(spyPaimonScanNode).getPaimonSplitFromAPI();
+ long maxInitialSplitSize = 32L * 1024L * 1024L;
+ long maxSplitSize = 64L * 1024L * 1024L;
+ // Ensure fileSplitter is initialized on the spy as doInitialize() is not called in this unit test
+ FileSplitter fileSplitter = new FileSplitter(maxInitialSplitSize, maxSplitSize,
+ 0);
+ try {
+ java.lang.reflect.Field field = FileQueryScanNode.class.getDeclaredField("fileSplitter");
+ field.setAccessible(true);
+ field.set(spyPaimonScanNode, fileSplitter);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException("Failed to inject FileSplitter into PaimonScanNode test", e);
+ }
+
// Note: The original PaimonSource is sufficient for this test
// No need to mock catalog properties since doInitialize() is not called in this test
// Mock SessionVariable behavior
Mockito.when(sv.isForceJniScanner()).thenReturn(false);
Mockito.when(sv.getIgnoreSplitType()).thenReturn("NONE");
+ Mockito.when(sv.getMaxInitialSplitSize()).thenReturn(maxInitialSplitSize);
+ Mockito.when(sv.getMaxSplitSize()).thenReturn(maxSplitSize);
// native
mockNativeReader(spyPaimonScanNode);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/FederationBackendPolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/FederationBackendPolicyTest.java
index 30582224f76..f6e8efd5294 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/FederationBackendPolicyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/FederationBackendPolicyTest.java
@@ -769,9 +769,9 @@ public class FederationBackendPolicyTest {
Map<Backend, List<Split>> backendListMap2 = mergeAssignment(assignment2);
backendListMap2.forEach((k, v) -> {
if (k.getId() == 1) {
- Assert.assertEquals(900000L, v.stream().mapToLong(Split::getLength).sum());
+ Assert.assertEquals(1000000L, v.stream().mapToLong(Split::getLength).sum());
} else if (k.getId() == 2) {
- Assert.assertEquals(500000L, v.stream().mapToLong(Split::getLength).sum());
+ Assert.assertEquals(400000L, v.stream().mapToLong(Split::getLength).sum());
} else if (k.getId() == 3) {
Assert.assertEquals(1000000L, v.stream().mapToLong(Split::getLength).sum());
}
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy b/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy
index 4117501eff2..44dd9104411 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_compress_type.groovy
@@ -48,7 +48,7 @@ suite("test_hive_compress_type", "p0,external,hive,external_docker,external_dock
sql """set file_split_size=8388608"""
explain {
sql("select count(*) from test_compress_partitioned")
- contains "inputSplitNum=82, totalFileSize=734675596, scanRanges=82"
+ contains "inputSplitNum=16, totalFileSize=734675596, scanRanges=16"
contains "partition=8/8"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/04: Manifest cache for tpch1000 (#59178)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bbca9eacc3b6c9fdd089500a03904f7240cd8259
Author: Socrates <su...@selectdb.com>
AuthorDate: Fri Dec 19 10:53:50 2025 +0800
Manifest cache for tpch1000 (#59178)
---
be/src/clucene | 1 +
fe/check/checkstyle/suppressions.xml | 3 +
.../main/java/org/apache/doris/common/Config.java | 12 +
.../doris/datasource/ExternalMetaCacheMgr.java | 7 +
.../iceberg/IcebergManifestCacheMgr.java | 35 +
.../doris/datasource/iceberg/IcebergUtils.java | 8 +
.../iceberg/cache/ContentFileEstimater.java | 194 +++++
.../iceberg/cache/IcebergManifestCache.java | 96 +++
.../iceberg/cache/IcebergManifestCacheLoader.java | 89 ++
.../datasource/iceberg/cache/ManifestCacheKey.java | 58 ++
.../iceberg/cache/ManifestCacheValue.java | 65 ++
.../datasource/iceberg/source/IcebergScanNode.java | 146 +++-
.../metastore/AbstractIcebergProperties.java | 62 ++
.../java/org/apache/iceberg/DeleteFileIndex.java | 906 +++++++++++++++++++++
14 files changed, 1681 insertions(+), 1 deletion(-)
diff --git a/be/src/clucene b/be/src/clucene
new file mode 160000
index 00000000000..bb22247973e
--- /dev/null
+++ b/be/src/clucene
@@ -0,0 +1 @@
+Subproject commit bb22247973e55dcac9a3eaafedc57cc6c36d2fc3
diff --git a/fe/check/checkstyle/suppressions.xml b/fe/check/checkstyle/suppressions.xml
index 8f000bb7616..7340c4c5bd5 100644
--- a/fe/check/checkstyle/suppressions.xml
+++ b/fe/check/checkstyle/suppressions.xml
@@ -69,6 +69,9 @@ under the License.
<!-- ignore hudi disk map copied from hudi/common/util/collection/DiskMap.java -->
<suppress files="org[\\/]apache[\\/]hudi[\\/]common[\\/]util[\\/]collection[\\/]DiskMap\.java" checks="[a-zA-Z0-9]*"/>
+ <!-- ignore iceberg delete file index copied from iceberg/DeleteFileIndex.java -->
+ <suppress files="org[\\/]apache[\\/]iceberg[\\/]DeleteFileIndex\.java" checks="[a-zA-Z0-9]*"/>
+
<!-- ignore gensrc/thrift/ExternalTableSchema.thrift -->
<suppress files=".*thrift/schema/external/.*" checks=".*"/>
</suppressions>
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 350e34f8a90..d4426c2d515 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2298,6 +2298,18 @@ public class Config extends ConfigBase {
})
public static long external_cache_refresh_time_minutes = 10; // 10 mins
+ @ConfField(description = {"是否启用 Iceberg Manifest DataFile/DeleteFile 缓存。",
+ "Whether to enable Iceberg manifest DataFile/DeleteFile cache."})
+ public static boolean iceberg_manifest_cache_enable = true;
+
+ @ConfField(description = {"Iceberg Manifest 缓存的容量上限,单位 MB。",
+ "Iceberg manifest cache capacity in MB."})
+ public static long iceberg_manifest_cache_capacity_mb = 1024;
+
+ @ConfField(description = {"Iceberg Manifest 缓存的访问过期时间(秒),0 或负数表示不过期。",
+ "Iceberg manifest cache expire after access in seconds. 0 or negative disables expiration."})
+ public static long iceberg_manifest_cache_ttl_sec = 48 * 60 * 60;
+
/**
* Github workflow test type, for setting some session variables
* only for certain test type. E.g. only settting batch_size to small
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index e777285a07f..798a2170b1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -29,6 +29,7 @@ import org.apache.doris.datasource.hudi.source.HudiCachedFsViewProcessor;
import org.apache.doris.datasource.hudi.source.HudiCachedMetaClientProcessor;
import org.apache.doris.datasource.hudi.source.HudiMetadataCacheMgr;
import org.apache.doris.datasource.hudi.source.HudiPartitionProcessor;
+import org.apache.doris.datasource.iceberg.IcebergManifestCacheMgr;
import org.apache.doris.datasource.iceberg.IcebergMetadataCache;
import org.apache.doris.datasource.iceberg.IcebergMetadataCacheMgr;
import org.apache.doris.datasource.maxcompute.MaxComputeMetadataCache;
@@ -97,6 +98,7 @@ public class ExternalMetaCacheMgr {
private FileSystemCache fsCache;
// all external table row count cache.
private ExternalRowCountCache rowCountCache;
+ private final IcebergManifestCacheMgr icebergManifestCacheMgr;
private final IcebergMetadataCacheMgr icebergMetadataCacheMgr;
private final MaxComputeMetadataCacheMgr maxComputeMetadataCacheMgr;
private final PaimonMetadataCacheMgr paimonMetadataCacheMgr;
@@ -128,6 +130,7 @@ public class ExternalMetaCacheMgr {
rowCountCache = new ExternalRowCountCache(rowCountRefreshExecutor);
hudiMetadataCacheMgr = new HudiMetadataCacheMgr(commonRefreshExecutor);
+ icebergManifestCacheMgr = new IcebergManifestCacheMgr();
icebergMetadataCacheMgr = new IcebergMetadataCacheMgr(commonRefreshExecutor);
maxComputeMetadataCacheMgr = new MaxComputeMetadataCacheMgr();
paimonMetadataCacheMgr = new PaimonMetadataCacheMgr(commonRefreshExecutor);
@@ -199,6 +202,10 @@ public class ExternalMetaCacheMgr {
return hudiMetadataCacheMgr;
}
+ public IcebergManifestCacheMgr getIcebergManifestCacheMgr() {
+ return icebergManifestCacheMgr;
+ }
+
public IcebergMetadataCache getIcebergMetadataCache() {
return icebergMetadataCacheMgr.getIcebergMetadataCache();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergManifestCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergManifestCacheMgr.java
new file mode 100644
index 00000000000..ad95e151b98
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergManifestCacheMgr.java
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg;
+
+import org.apache.doris.datasource.iceberg.cache.IcebergManifestCache;
+
+/**
+ * Wrapper manager for Iceberg manifest cache.
+ */
+public class IcebergManifestCacheMgr {
+ private final IcebergManifestCache manifestCache;
+
+ public IcebergManifestCacheMgr() {
+ this.manifestCache = new IcebergManifestCache();
+ }
+
+ public IcebergManifestCache getManifestCache() {
+ return manifestCache;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index 9587ca4f816..28ddf2817df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -56,6 +56,7 @@ import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalSchemaCache;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
+import org.apache.doris.datasource.iceberg.cache.IcebergManifestCache;
import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo;
import org.apache.doris.datasource.mvcc.MvccSnapshot;
import org.apache.doris.datasource.mvcc.MvccUtil;
@@ -1452,4 +1453,11 @@ public class IcebergUtils {
icebergExternalTable.getViewText();
}
+ public static IcebergManifestCache getManifestCache() {
+ return Env.getCurrentEnv()
+ .getExtMetaCacheMgr()
+ .getIcebergManifestCacheMgr()
+ .getManifestCache();
+ }
+
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ContentFileEstimater.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ContentFileEstimater.java
new file mode 100644
index 00000000000..43f60096e31
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ContentFileEstimater.java
@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.cache;
+
+import org.apache.iceberg.ContentFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.StructLike;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Utility to estimate the JVM weight of Iceberg {@link ContentFile} objects.
+ */
+public final class ContentFileEstimater {
+ private static final long LIST_BASE_WEIGHT = 48L;
+ private static final long OBJECT_REFERENCE_WEIGHT = 8L;
+ private static final long CONTENT_FILE_BASE_WEIGHT = 256L;
+ private static final long STRING_BASE_WEIGHT = 40L;
+ private static final long CHAR_BYTES = 2L;
+ private static final long BYTE_BUFFER_BASE_WEIGHT = 16L;
+ private static final long MAP_BASE_WEIGHT = 48L;
+ private static final long MAP_ENTRY_OVERHEAD = 24L;
+ private static final long LONG_OBJECT_WEIGHT = 24L;
+ private static final long INT_OBJECT_WEIGHT = 16L;
+ private static final long PARTITION_BASE_WEIGHT = 48L;
+ private static final long PARTITION_VALUE_BASE_WEIGHT = 8L;
+
+ private ContentFileEstimater() {
+ }
+
+ public static long estimate(List<? extends ContentFile<?>> files) {
+ return listReferenceWeight(files) + estimateContentFilesWeight(files);
+ }
+
+ private static long listReferenceWeight(List<?> files) {
+ if (files == null || files.isEmpty()) {
+ return 0L;
+ }
+ return LIST_BASE_WEIGHT + (long) files.size() * OBJECT_REFERENCE_WEIGHT;
+ }
+
+ private static long estimateContentFilesWeight(List<? extends ContentFile<?>> files) {
+ long total = 0L;
+ if (files == null) {
+ return 0L;
+ }
+ for (ContentFile<?> file : files) {
+ total += estimateContentFileWeight(file);
+ }
+ return total;
+ }
+
+ private static long estimateContentFileWeight(ContentFile<?> file) {
+ if (file == null) {
+ return 0L;
+ }
+
+ long weight = CONTENT_FILE_BASE_WEIGHT;
+ weight += charSequenceWeight(file.path());
+ weight += stringWeight(file.manifestLocation());
+ weight += byteBufferWeight(file.keyMetadata());
+ weight += partitionWeight(file.partition());
+
+ weight += numericMapWeight(file.columnSizes());
+ weight += numericMapWeight(file.valueCounts());
+ weight += numericMapWeight(file.nullValueCounts());
+ weight += numericMapWeight(file.nanValueCounts());
+ weight += byteBufferMapWeight(file.lowerBounds());
+ weight += byteBufferMapWeight(file.upperBounds());
+
+ weight += listWeight(file.splitOffsets(), LONG_OBJECT_WEIGHT);
+ weight += listWeight(file.equalityFieldIds(), INT_OBJECT_WEIGHT);
+
+ weight += optionalLongWeight(file.pos());
+ weight += optionalLongWeight(file.dataSequenceNumber());
+ weight += optionalLongWeight(file.fileSequenceNumber());
+ weight += optionalLongWeight(file.firstRowId());
+ weight += optionalIntWeight(file.sortOrderId());
+
+ if (file instanceof DeleteFile) {
+ DeleteFile deleteFile = (DeleteFile) file;
+ weight += stringWeight(deleteFile.referencedDataFile());
+ weight += optionalLongWeight(deleteFile.contentOffset());
+ weight += optionalLongWeight(deleteFile.contentSizeInBytes());
+ }
+
+ return weight;
+ }
+
+ private static long listWeight(List<? extends Number> list, long elementWeight) {
+ if (list == null || list.isEmpty()) {
+ return 0L;
+ }
+ return LIST_BASE_WEIGHT + (long) list.size() * (OBJECT_REFERENCE_WEIGHT + elementWeight);
+ }
+
+ private static long numericMapWeight(Map<Integer, Long> map) {
+ if (map == null || map.isEmpty()) {
+ return 0L;
+ }
+ return MAP_BASE_WEIGHT + (long) map.size() * (MAP_ENTRY_OVERHEAD + LONG_OBJECT_WEIGHT);
+ }
+
+ private static long byteBufferMapWeight(Map<Integer, ByteBuffer> map) {
+ if (map == null || map.isEmpty()) {
+ return 0L;
+ }
+ long weight = MAP_BASE_WEIGHT + (long) map.size() * MAP_ENTRY_OVERHEAD;
+ for (ByteBuffer buffer : map.values()) {
+ weight += byteBufferWeight(buffer);
+ }
+ return weight;
+ }
+
+ private static long partitionWeight(StructLike partition) {
+ if (partition == null) {
+ return 0L;
+ }
+ long weight = PARTITION_BASE_WEIGHT + (long) partition.size() * PARTITION_VALUE_BASE_WEIGHT;
+ for (int i = 0; i < partition.size(); i++) {
+ Object value = partition.get(i, Object.class);
+ weight += estimateValueWeight(value);
+ }
+ return weight;
+ }
+
+ private static long estimateValueWeight(Object value) {
+ if (value == null) {
+ return 0L;
+ }
+ if (value instanceof CharSequence) {
+ return charSequenceWeight((CharSequence) value);
+ } else if (value instanceof byte[]) {
+ return BYTE_BUFFER_BASE_WEIGHT + ((byte[]) value).length;
+ } else if (value instanceof ByteBuffer) {
+ return byteBufferWeight((ByteBuffer) value);
+ } else if (value instanceof Long || value instanceof Double) {
+ return LONG_OBJECT_WEIGHT;
+ } else if (value instanceof Integer || value instanceof Float) {
+ return INT_OBJECT_WEIGHT;
+ } else if (value instanceof Short || value instanceof Character) {
+ return 4L;
+ } else if (value instanceof Boolean) {
+ return 1L;
+ }
+ return OBJECT_REFERENCE_WEIGHT;
+ }
+
+ private static long charSequenceWeight(CharSequence value) {
+ if (value == null) {
+ return 0L;
+ }
+ return STRING_BASE_WEIGHT + (long) value.length() * CHAR_BYTES;
+ }
+
+ private static long stringWeight(String value) {
+ if (value == null) {
+ return 0L;
+ }
+ return STRING_BASE_WEIGHT + (long) value.length() * CHAR_BYTES;
+ }
+
+ private static long byteBufferWeight(ByteBuffer buffer) {
+ if (buffer == null) {
+ return 0L;
+ }
+ return BYTE_BUFFER_BASE_WEIGHT + buffer.remaining();
+ }
+
+ private static long optionalLongWeight(Long value) {
+ return value == null ? 0L : LONG_OBJECT_WEIGHT;
+ }
+
+ private static long optionalIntWeight(Integer value) {
+ return value == null ? 0L : INT_OBJECT_WEIGHT;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCache.java
new file mode 100644
index 00000000000..be919c5d313
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCache.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.cache;
+
+import org.apache.doris.common.Config;
+import org.apache.doris.datasource.CacheException;
+
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.github.benmanes.caffeine.cache.Weigher;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+
+/**
+ * A lightweight manifest cache that stores parsed DataFile/DeleteFile lists per manifest.
+ */
+public class IcebergManifestCache {
+ private static final Logger LOG = LogManager.getLogger(IcebergManifestCache.class);
+
+ private final LoadingCache<ManifestCacheKey, ManifestCacheValue> cache;
+
+ public IcebergManifestCache() {
+ long capacityInBytes = Config.iceberg_manifest_cache_capacity_mb * 1024L * 1024L;
+ Weigher<ManifestCacheKey, ManifestCacheValue> weigher = (key, value) -> {
+ long weight = Optional.ofNullable(value).map(ManifestCacheValue::getWeightBytes).orElse(0L);
+ if (weight > Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ }
+ return (int) weight;
+ };
+ Caffeine<ManifestCacheKey, ManifestCacheValue> builder = Caffeine.newBuilder()
+ .maximumWeight(capacityInBytes)
+ .weigher(weigher);
+ if (Config.iceberg_manifest_cache_ttl_sec > 0) {
+ builder = builder.expireAfterAccess(Duration.ofSeconds(Config.iceberg_manifest_cache_ttl_sec));
+ }
+ cache = builder.build(new CacheLoader<ManifestCacheKey, ManifestCacheValue>() {
+ @Override
+ public ManifestCacheValue load(ManifestCacheKey key) {
+ throw new CacheException("Manifest cache loader should be provided explicitly for key %s", null, key);
+ }
+ });
+ }
+
+ public ManifestCacheValue get(ManifestCacheKey key, Callable<ManifestCacheValue> loader) {
+ try {
+ return cache.get(key, ignored -> {
+ try {
+ return loader.call();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ } catch (Exception e) {
+ throw new CacheException("Failed to load manifest cache for key %s", e, key);
+ }
+ }
+
+ public Optional<ManifestCacheValue> peek(ManifestCacheKey key) {
+ return Optional.ofNullable(cache.getIfPresent(key));
+ }
+
+ public void invalidateByPath(String path) {
+ cache.asMap().keySet().stream()
+ .filter(key -> key.getPath().equals(path))
+ .forEach(cache::invalidate);
+ }
+
+ public void invalidateAll() {
+ cache.invalidateAll();
+ }
+
+ public ManifestCacheKey buildKey(String path) {
+ return new ManifestCacheKey(path);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCacheLoader.java
new file mode 100644
index 00000000000..dc4d16da61b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCacheLoader.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.cache;
+
+import org.apache.doris.datasource.CacheException;
+
+import com.google.common.collect.Lists;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
+import org.apache.iceberg.Table;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Helper to load manifest content and populate the manifest cache.
+ */
+public class IcebergManifestCacheLoader {
+ private static final Logger LOG = LogManager.getLogger(IcebergManifestCacheLoader.class);
+
+ private IcebergManifestCacheLoader() {
+ }
+
+ public static ManifestCacheValue loadDataFilesWithCache(IcebergManifestCache cache, ManifestFile manifest,
+ Table table) {
+ ManifestCacheKey key = buildKey(cache, manifest);
+ return cache.get(key, () -> loadDataFiles(manifest, table));
+ }
+
+ public static ManifestCacheValue loadDeleteFilesWithCache(IcebergManifestCache cache, ManifestFile manifest,
+ Table table) {
+ ManifestCacheKey key = buildKey(cache, manifest);
+ return cache.get(key, () -> loadDeleteFiles(manifest, table));
+ }
+
+ private static ManifestCacheValue loadDataFiles(ManifestFile manifest, Table table) {
+ List<DataFile> dataFiles = Lists.newArrayList();
+ try (ManifestReader<DataFile> reader = ManifestFiles.read(manifest, table.io())) {
+ // ManifestReader implements CloseableIterable<DataFile>, iterate directly
+ for (DataFile dataFile : reader) {
+ dataFiles.add(dataFile.copy());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to read data manifest {}", manifest.path(), e);
+ throw new CacheException("Failed to read data manifest %s", e, manifest.path());
+ }
+ return ManifestCacheValue.forDataFiles(dataFiles);
+ }
+
+ private static ManifestCacheValue loadDeleteFiles(ManifestFile manifest, Table table) {
+ List<DeleteFile> deleteFiles = Lists.newArrayList();
+ try (ManifestReader<DeleteFile> reader = ManifestFiles.readDeleteManifest(manifest, table.io(),
+ table.specs())) {
+ // ManifestReader implements CloseableIterable<DeleteFile>, iterate directly
+ for (DeleteFile deleteFile : reader) {
+ deleteFiles.add(deleteFile.copy());
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to read delete manifest {}", manifest.path(), e);
+ throw new CacheException("Failed to read delete manifest %s", e, manifest.path());
+ }
+ return ManifestCacheValue.forDeleteFiles(deleteFiles);
+ }
+
+ private static ManifestCacheKey buildKey(IcebergManifestCache cache, ManifestFile manifest) {
+ // Iceberg manifest files are immutable, so path uniquely identifies a manifest
+ return cache.buildKey(manifest.path());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheKey.java
new file mode 100644
index 00000000000..41b52187aec
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheKey.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.cache;
+
+import java.util.Objects;
+
+/**
+ * Cache key for a single Iceberg manifest file.
+ * Since Iceberg manifest files are immutable, path uniquely identifies a manifest.
+ */
+public class ManifestCacheKey {
+ private final String path;
+
+ public ManifestCacheKey(String path) {
+ this.path = path;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ManifestCacheKey)) {
+ return false;
+ }
+ ManifestCacheKey that = (ManifestCacheKey) o;
+ return Objects.equals(path, that.path);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path);
+ }
+
+ @Override
+ public String toString() {
+ return "ManifestCacheKey{path='" + path + "'}";
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java
new file mode 100644
index 00000000000..0c7c9154639
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.cache;
+
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Cached manifest payload containing parsed files and an estimated weight.
+ */
+public class ManifestCacheValue {
+ private final List<DataFile> dataFiles;
+ private final List<DeleteFile> deleteFiles;
+ private final long weightBytes;
+
+ private ManifestCacheValue(List<DataFile> dataFiles, List<DeleteFile> deleteFiles, long weightBytes) {
+ this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles;
+ this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles;
+ this.weightBytes = weightBytes;
+ }
+
+ public static ManifestCacheValue forDataFiles(List<DataFile> dataFiles) {
+ return new ManifestCacheValue(dataFiles, Collections.emptyList(),
+ estimateWeight(dataFiles, Collections.emptyList()));
+ }
+
+ public static ManifestCacheValue forDeleteFiles(List<DeleteFile> deleteFiles) {
+ return new ManifestCacheValue(Collections.emptyList(), deleteFiles,
+ estimateWeight(Collections.emptyList(), deleteFiles));
+ }
+
+ public List<DataFile> getDataFiles() {
+ return dataFiles;
+ }
+
+ public List<DeleteFile> getDeleteFiles() {
+ return deleteFiles;
+ }
+
+ public long getWeightBytes() {
+ return weightBytes;
+ }
+
+ private static long estimateWeight(List<DataFile> dataFiles, List<DeleteFile> deleteFiles) {
+ return ContentFileEstimater.estimate(dataFiles) + ContentFileEstimater.estimate(deleteFiles);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index f5208397a0f..0ffe86edb31 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
@@ -38,6 +39,9 @@ import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.datasource.iceberg.cache.IcebergManifestCache;
+import org.apache.doris.datasource.iceberg.cache.IcebergManifestCacheLoader;
+import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.nereids.exceptions.NotSupportedException;
import org.apache.doris.planner.PlanNodeId;
@@ -57,18 +61,27 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.iceberg.BaseFileScanTask;
import org.apache.iceberg.BaseTable;
import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.DeleteFileIndex;
import org.apache.iceberg.FileContent;
import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.ManifestContent;
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.PartitionData;
import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.PartitionSpecParser;
+import org.apache.iceberg.SchemaParser;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableScan;
import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.expressions.InclusiveMetricsEvaluator;
+import org.apache.iceberg.expressions.ManifestEvaluator;
+import org.apache.iceberg.expressions.ResidualEvaluator;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.CloseableIterator;
import org.apache.iceberg.types.Conversions;
@@ -78,9 +91,12 @@ import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.concurrent.CompletableFuture;
@@ -358,8 +374,136 @@ public class IcebergScanNode extends FileQueryScanNode {
}
private CloseableIterable<FileScanTask> planFileScanTask(TableScan scan) {
+ if (!Config.iceberg_manifest_cache_enable) {
+ long targetSplitSize = getRealFileSplitSize(0);
+ return TableScanUtil.splitFiles(scan.planFiles(), targetSplitSize);
+ }
+ try {
+ return planFileScanTaskWithManifestCache(scan);
+ } catch (Exception e) {
+ LOG.warn("Plan with manifest cache failed, fallback to original scan: {}", e.getMessage());
+ long targetSplitSize = getRealFileSplitSize(0);
+ return TableScanUtil.splitFiles(scan.planFiles(), targetSplitSize);
+ }
+ }
+
+ private CloseableIterable<FileScanTask> planFileScanTaskWithManifestCache(TableScan scan) throws IOException {
+ // Get the snapshot from the scan; return empty if no snapshot exists
+ Snapshot snapshot = scan.snapshot();
+ if (snapshot == null) {
+ return CloseableIterable.withNoopClose(Collections.emptyList());
+ }
+
+ // Initialize manifest cache for efficient manifest file access
+ IcebergManifestCache cache = IcebergUtils.getManifestCache();
+
+ // Convert query conjuncts to Iceberg filter expression
+ // This combines all predicates with AND logic for partition/file pruning
+ Expression filterExpr = conjuncts.stream()
+ .map(conjunct -> IcebergUtils.convertToIcebergExpr(conjunct, icebergTable.schema()))
+ .filter(Objects::nonNull)
+ .reduce(Expressions.alwaysTrue(), Expressions::and);
+
+ // Get all partition specs by their IDs for later use
+ Map<Integer, PartitionSpec> specsById = icebergTable.specs();
+ boolean caseSensitive = true;
+
+ // Create residual evaluators for each partition spec
+ // Residual evaluators compute the remaining filter expression after partition pruning
+ Map<Integer, ResidualEvaluator> residualEvaluators = new HashMap<>();
+ specsById.forEach((id, spec) -> residualEvaluators.put(id,
+ ResidualEvaluator.of(spec, filterExpr == null ? Expressions.alwaysTrue() : filterExpr,
+ caseSensitive)));
+
+ // Create metrics evaluator for file-level pruning based on column statistics
+ InclusiveMetricsEvaluator metricsEvaluator = filterExpr == null ? null
+ : new InclusiveMetricsEvaluator(icebergTable.schema(), filterExpr, caseSensitive);
+
+ // ========== Phase 1: Load delete files from delete manifests ==========
+ List<DeleteFile> deleteFiles = new ArrayList<>();
+ List<ManifestFile> deleteManifests = snapshot.deleteManifests(icebergTable.io());
+ for (ManifestFile manifest : deleteManifests) {
+ // Skip non-delete manifests
+ if (manifest.content() != ManifestContent.DELETES) {
+ continue;
+ }
+ // Get the partition spec for this manifest
+ PartitionSpec spec = specsById.get(manifest.partitionSpecId());
+ if (spec == null) {
+ continue;
+ }
+ // Create manifest evaluator for partition-level pruning
+ ManifestEvaluator evaluator = filterExpr == null ? null
+ : ManifestEvaluator.forPartitionFilter(filterExpr, spec, caseSensitive);
+ // Skip manifest if it doesn't match the filter expression (partition pruning)
+ if (evaluator != null && !evaluator.eval(manifest)) {
+ continue;
+ }
+ // Load delete files from cache (or from storage if not cached)
+ ManifestCacheValue value = IcebergManifestCacheLoader.loadDeleteFilesWithCache(cache, manifest,
+ icebergTable);
+ deleteFiles.addAll(value.getDeleteFiles());
+ }
+
+ // Build delete file index for efficient lookup of deletes applicable to each data file
+ DeleteFileIndex deleteIndex = DeleteFileIndex.builderFor(deleteFiles)
+ .specsById(specsById)
+ .caseSensitive(caseSensitive)
+ .build();
+
+ // ========== Phase 2: Load data files and create scan tasks ==========
+ List<FileScanTask> tasks = new ArrayList<>();
+ try (CloseableIterable<ManifestFile> dataManifests =
+ IcebergUtils.getMatchingManifest(snapshot.dataManifests(icebergTable.io()),
+ specsById, filterExpr)) {
+ for (ManifestFile manifest : dataManifests) {
+ // Skip non-data manifests
+ if (manifest.content() != ManifestContent.DATA) {
+ continue;
+ }
+ // Get the partition spec for this manifest
+ PartitionSpec spec = specsById.get(manifest.partitionSpecId());
+ if (spec == null) {
+ continue;
+ }
+ // Get the residual evaluator for this partition spec
+ ResidualEvaluator residualEvaluator = residualEvaluators.get(manifest.partitionSpecId());
+
+ // Load data files from cache (or from storage if not cached)
+ ManifestCacheValue value = IcebergManifestCacheLoader.loadDataFilesWithCache(cache, manifest,
+ icebergTable);
+
+ // Process each data file in the manifest
+ for (org.apache.iceberg.DataFile dataFile : value.getDataFiles()) {
+ // Skip file if column statistics indicate no matching rows (metrics-based pruning)
+ if (metricsEvaluator != null && !metricsEvaluator.eval(dataFile)) {
+ continue;
+ }
+ // Skip file if partition values don't match the residual filter
+ if (residualEvaluator != null) {
+ if (residualEvaluator.residualFor(dataFile.partition()).equals(Expressions.alwaysFalse())) {
+ continue;
+ }
+ }
+ // Find all delete files that apply to this data file based on sequence number
+ List<DeleteFile> deletes = Arrays.asList(
+ deleteIndex.forDataFile(dataFile.dataSequenceNumber(), dataFile));
+
+ // Create a FileScanTask containing the data file, associated deletes, and metadata
+ tasks.add(new BaseFileScanTask(
+ dataFile,
+ deletes.toArray(new DeleteFile[0]),
+ SchemaParser.toJson(icebergTable.schema()),
+ PartitionSpecParser.toJson(spec),
+ residualEvaluator == null ? ResidualEvaluator.unpartitioned(Expressions.alwaysTrue())
+ : residualEvaluator));
+ }
+ }
+ }
+
+ // Split tasks into smaller chunks based on target split size for parallel processing
long targetSplitSize = getRealFileSplitSize(0);
- return TableScanUtil.splitFiles(scan.planFiles(), targetSplitSize);
+ return TableScanUtil.splitFiles(CloseableIterable.withNoopClose(tasks), targetSplitSize);
}
private Split createIcebergSplit(FileScanTask fileScanTask) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
index 2cc829c8743..88def12d2a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
@@ -43,6 +43,43 @@ public abstract class AbstractIcebergProperties extends MetastoreProperties {
)
protected String warehouse;
+ @Getter
+ @ConnectorProperty(
+ names = {CatalogProperties.IO_MANIFEST_CACHE_ENABLED},
+ required = false,
+ description = "Controls whether to use caching during manifest reads or not. Default: false."
+ )
+ protected String ioManifestCacheEnabled;
+
+ @Getter
+ @ConnectorProperty(
+ names = {CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS},
+ required = false,
+ description = "Controls the maximum duration for which an entry stays in the manifest cache. "
+ + "Must be a non-negative value. Zero means entries expire only due to memory pressure. "
+ + "Default: 60000 (60s)."
+ )
+ protected String ioManifestCacheExpirationIntervalMs;
+
+ @Getter
+ @ConnectorProperty(
+ names = {CatalogProperties.IO_MANIFEST_CACHE_MAX_TOTAL_BYTES},
+ required = false,
+ description = "Controls the maximum total amount of bytes to cache in manifest cache. "
+ + "Must be a positive value. Default: 104857600 (100MB)."
+ )
+ protected String ioManifestCacheMaxTotalBytes;
+
+ @Getter
+ @ConnectorProperty(
+ names = {CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH},
+ required = false,
+ description = "Controls the maximum length of file to be considered for caching. "
+ + "An InputFile will not be cached if the length is longer than this limit. "
+ + "Must be a positive value. Default: 8388608 (8MB)."
+ )
+ protected String ioManifestCacheMaxContentLength;
+
@Getter
protected ExecutionAuthenticator executionAuthenticator = new ExecutionAuthenticator(){};
@@ -80,6 +117,9 @@ public abstract class AbstractIcebergProperties extends MetastoreProperties {
catalogProps.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
}
+ // Add manifest cache properties if configured
+ addManifestCacheProperties(catalogProps);
+
Catalog catalog = initCatalog(catalogName, catalogProps, storagePropertiesList);
if (catalog == null) {
@@ -88,6 +128,28 @@ public abstract class AbstractIcebergProperties extends MetastoreProperties {
return catalog;
}
+ /**
+ * Add manifest cache related properties to catalog properties.
+ * These properties control caching behavior during manifest reads.
+ *
+ * @param catalogProps the catalog properties map to add manifest cache properties to
+ */
+ protected void addManifestCacheProperties(Map<String, String> catalogProps) {
+ if (StringUtils.isNotBlank(ioManifestCacheEnabled)) {
+ catalogProps.put(CatalogProperties.IO_MANIFEST_CACHE_ENABLED, ioManifestCacheEnabled);
+ }
+ if (StringUtils.isNotBlank(ioManifestCacheExpirationIntervalMs)) {
+ catalogProps.put(CatalogProperties.IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS,
+ ioManifestCacheExpirationIntervalMs);
+ }
+ if (StringUtils.isNotBlank(ioManifestCacheMaxTotalBytes)) {
+ catalogProps.put(CatalogProperties.IO_MANIFEST_CACHE_MAX_TOTAL_BYTES, ioManifestCacheMaxTotalBytes);
+ }
+ if (StringUtils.isNotBlank(ioManifestCacheMaxContentLength)) {
+ catalogProps.put(CatalogProperties.IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, ioManifestCacheMaxContentLength);
+ }
+ }
+
/**
* Subclasses must implement this to create the concrete Catalog instance.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java b/fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
new file mode 100644
index 00000000000..5c9cdd93c45
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
@@ -0,0 +1,906 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.expressions.ManifestEvaluator;
+import org.apache.iceberg.expressions.Projections;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.metrics.ScanMetrics;
+import org.apache.iceberg.metrics.ScanMetricsUtil;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.ArrayUtil;
+import org.apache.iceberg.util.ContentFileUtil;
+import org.apache.iceberg.util.PartitionMap;
+import org.apache.iceberg.util.PartitionSet;
+import org.apache.iceberg.util.Tasks;
+
+/**
+ * An index of {@link DeleteFile delete files} by sequence number.
+ *
+ * <p>Use {@link #builderFor(FileIO, Iterable)} to construct an index, and {@link #forDataFile(long,
+ * DataFile)} or {@link #forEntry(ManifestEntry)} to get the delete files to apply to a given data
+ * file.
+ *
+ * Copyed from https://github.com/apache/iceberg/blob/apache-iceberg-1.9.1/core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
+ * Change DeleteFileIndex and some methods to public.
+ */
+public class DeleteFileIndex {
+ private static final DeleteFile[] EMPTY_DELETES = new DeleteFile[0];
+
+ private final EqualityDeletes globalDeletes;
+ private final PartitionMap<EqualityDeletes> eqDeletesByPartition;
+ private final PartitionMap<PositionDeletes> posDeletesByPartition;
+ private final Map<String, PositionDeletes> posDeletesByPath;
+ private final Map<String, DeleteFile> dvByPath;
+ private final boolean hasEqDeletes;
+ private final boolean hasPosDeletes;
+ private final boolean isEmpty;
+
+ private DeleteFileIndex(
+ EqualityDeletes globalDeletes,
+ PartitionMap<EqualityDeletes> eqDeletesByPartition,
+ PartitionMap<PositionDeletes> posDeletesByPartition,
+ Map<String, PositionDeletes> posDeletesByPath,
+ Map<String, DeleteFile> dvByPath) {
+ this.globalDeletes = globalDeletes;
+ this.eqDeletesByPartition = eqDeletesByPartition;
+ this.posDeletesByPartition = posDeletesByPartition;
+ this.posDeletesByPath = posDeletesByPath;
+ this.dvByPath = dvByPath;
+ this.hasEqDeletes = globalDeletes != null || eqDeletesByPartition != null;
+ this.hasPosDeletes =
+ posDeletesByPartition != null || posDeletesByPath != null || dvByPath != null;
+ this.isEmpty = !hasEqDeletes && !hasPosDeletes;
+ }
+
+ public boolean isEmpty() {
+ return isEmpty;
+ }
+
+ public boolean hasEqualityDeletes() {
+ return hasEqDeletes;
+ }
+
+ public boolean hasPositionDeletes() {
+ return hasPosDeletes;
+ }
+
+ public Iterable<DeleteFile> referencedDeleteFiles() {
+ Iterable<DeleteFile> deleteFiles = Collections.emptyList();
+
+ if (globalDeletes != null) {
+ deleteFiles = Iterables.concat(deleteFiles, globalDeletes.referencedDeleteFiles());
+ }
+
+ if (eqDeletesByPartition != null) {
+ for (EqualityDeletes deletes : eqDeletesByPartition.values()) {
+ deleteFiles = Iterables.concat(deleteFiles, deletes.referencedDeleteFiles());
+ }
+ }
+
+ if (posDeletesByPartition != null) {
+ for (PositionDeletes deletes : posDeletesByPartition.values()) {
+ deleteFiles = Iterables.concat(deleteFiles, deletes.referencedDeleteFiles());
+ }
+ }
+
+ if (posDeletesByPath != null) {
+ for (PositionDeletes deletes : posDeletesByPath.values()) {
+ deleteFiles = Iterables.concat(deleteFiles, deletes.referencedDeleteFiles());
+ }
+ }
+
+ if (dvByPath != null) {
+ deleteFiles = Iterables.concat(deleteFiles, dvByPath.values());
+ }
+
+ return deleteFiles;
+ }
+
+ DeleteFile[] forEntry(ManifestEntry<DataFile> entry) {
+ return forDataFile(entry.dataSequenceNumber(), entry.file());
+ }
+
+ public DeleteFile[] forDataFile(DataFile file) {
+ return forDataFile(file.dataSequenceNumber(), file);
+ }
+
+ public DeleteFile[] forDataFile(long sequenceNumber, DataFile file) {
+ if (isEmpty) {
+ return EMPTY_DELETES;
+ }
+
+ DeleteFile[] global = findGlobalDeletes(sequenceNumber, file);
+ DeleteFile[] eqPartition = findEqPartitionDeletes(sequenceNumber, file);
+ DeleteFile dv = findDV(sequenceNumber, file);
+ if (dv != null && global == null && eqPartition == null) {
+ return new DeleteFile[] {dv};
+ } else if (dv != null) {
+ return concat(global, eqPartition, new DeleteFile[] {dv});
+ } else {
+ DeleteFile[] posPartition = findPosPartitionDeletes(sequenceNumber, file);
+ DeleteFile[] posPath = findPathDeletes(sequenceNumber, file);
+ return concat(global, eqPartition, posPartition, posPath);
+ }
+ }
+
+ private DeleteFile[] findGlobalDeletes(long seq, DataFile dataFile) {
+ return globalDeletes == null ? EMPTY_DELETES : globalDeletes.filter(seq, dataFile);
+ }
+
+ private DeleteFile[] findPosPartitionDeletes(long seq, DataFile dataFile) {
+ if (posDeletesByPartition == null) {
+ return EMPTY_DELETES;
+ }
+
+ PositionDeletes deletes = posDeletesByPartition.get(dataFile.specId(), dataFile.partition());
+ return deletes == null ? EMPTY_DELETES : deletes.filter(seq);
+ }
+
+ private DeleteFile[] findEqPartitionDeletes(long seq, DataFile dataFile) {
+ if (eqDeletesByPartition == null) {
+ return EMPTY_DELETES;
+ }
+
+ EqualityDeletes deletes = eqDeletesByPartition.get(dataFile.specId(), dataFile.partition());
+ return deletes == null ? EMPTY_DELETES : deletes.filter(seq, dataFile);
+ }
+
+ @SuppressWarnings("CollectionUndefinedEquality")
+ private DeleteFile[] findPathDeletes(long seq, DataFile dataFile) {
+ if (posDeletesByPath == null) {
+ return EMPTY_DELETES;
+ }
+
+ PositionDeletes deletes = posDeletesByPath.get(dataFile.location());
+ return deletes == null ? EMPTY_DELETES : deletes.filter(seq);
+ }
+
+ private DeleteFile findDV(long seq, DataFile dataFile) {
+ if (dvByPath == null) {
+ return null;
+ }
+
+ DeleteFile dv = dvByPath.get(dataFile.location());
+ if (dv != null) {
+ ValidationException.check(
+ dv.dataSequenceNumber() >= seq,
+ "DV data sequence number (%s) must be greater than or equal to data file sequence number (%s)",
+ dv.dataSequenceNumber(),
+ seq);
+ }
+ return dv;
+ }
+
+ @SuppressWarnings("checkstyle:CyclomaticComplexity")
+ private static boolean canContainEqDeletesForFile(
+ DataFile dataFile, EqualityDeleteFile deleteFile) {
+ Map<Integer, ByteBuffer> dataLowers = dataFile.lowerBounds();
+ Map<Integer, ByteBuffer> dataUppers = dataFile.upperBounds();
+
+ // whether to check data ranges or to assume that the ranges match
+ // if upper/lower bounds are missing, null counts may still be used to determine delete files
+ // can be skipped
+ boolean checkRanges =
+ dataLowers != null && dataUppers != null && deleteFile.hasLowerAndUpperBounds();
+
+ Map<Integer, Long> dataNullCounts = dataFile.nullValueCounts();
+ Map<Integer, Long> dataValueCounts = dataFile.valueCounts();
+ Map<Integer, Long> deleteNullCounts = deleteFile.nullValueCounts();
+ Map<Integer, Long> deleteValueCounts = deleteFile.valueCounts();
+
+ for (Types.NestedField field : deleteFile.equalityFields()) {
+ if (!field.type().isPrimitiveType()) {
+ // stats are not kept for nested types. assume that the delete file may match
+ continue;
+ }
+
+ if (containsNull(dataNullCounts, field) && containsNull(deleteNullCounts, field)) {
+ // the data has null values and null has been deleted, so the deletes must be applied
+ continue;
+ }
+
+ if (allNull(dataNullCounts, dataValueCounts, field) && allNonNull(deleteNullCounts, field)) {
+ // the data file contains only null values for this field, but there are no deletes for null
+ // values
+ return false;
+ }
+
+ if (allNull(deleteNullCounts, deleteValueCounts, field)
+ && allNonNull(dataNullCounts, field)) {
+ // the delete file removes only null rows with null for this field, but there are no data
+ // rows with null
+ return false;
+ }
+
+ if (!checkRanges) {
+ // some upper and lower bounds are missing, assume they match
+ continue;
+ }
+
+ int id = field.fieldId();
+ ByteBuffer dataLower = dataLowers.get(id);
+ ByteBuffer dataUpper = dataUppers.get(id);
+ Object deleteLower = deleteFile.lowerBound(id);
+ Object deleteUpper = deleteFile.upperBound(id);
+ if (dataLower == null || dataUpper == null || deleteLower == null || deleteUpper == null) {
+ // at least one bound is not known, assume the delete file may match
+ continue;
+ }
+
+ if (!rangesOverlap(field, dataLower, dataUpper, deleteLower, deleteUpper)) {
+ // no values overlap between the data file and the deletes
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private static <T> boolean rangesOverlap(
+ Types.NestedField field,
+ ByteBuffer dataLowerBuf,
+ ByteBuffer dataUpperBuf,
+ T deleteLower,
+ T deleteUpper) {
+ Type.PrimitiveType type = field.type().asPrimitiveType();
+ Comparator<T> comparator = Comparators.forType(type);
+
+ T dataLower = Conversions.fromByteBuffer(type, dataLowerBuf);
+ if (comparator.compare(dataLower, deleteUpper) > 0) {
+ return false;
+ }
+
+ T dataUpper = Conversions.fromByteBuffer(type, dataUpperBuf);
+ if (comparator.compare(deleteLower, dataUpper) > 0) {
+ return false;
+ }
+
+ return true;
+ }
+
+ private static boolean allNonNull(Map<Integer, Long> nullValueCounts, Types.NestedField field) {
+ if (field.isRequired()) {
+ return true;
+ }
+
+ if (nullValueCounts == null) {
+ return false;
+ }
+
+ Long nullValueCount = nullValueCounts.get(field.fieldId());
+ if (nullValueCount == null) {
+ return false;
+ }
+
+ return nullValueCount <= 0;
+ }
+
+ private static boolean allNull(
+ Map<Integer, Long> nullValueCounts, Map<Integer, Long> valueCounts, Types.NestedField field) {
+ if (field.isRequired()) {
+ return false;
+ }
+
+ if (nullValueCounts == null || valueCounts == null) {
+ return false;
+ }
+
+ Long nullValueCount = nullValueCounts.get(field.fieldId());
+ Long valueCount = valueCounts.get(field.fieldId());
+ if (nullValueCount == null || valueCount == null) {
+ return false;
+ }
+
+ return nullValueCount.equals(valueCount);
+ }
+
+ private static boolean containsNull(Map<Integer, Long> nullValueCounts, Types.NestedField field) {
+ if (field.isRequired()) {
+ return false;
+ }
+
+ if (nullValueCounts == null) {
+ return true;
+ }
+
+ Long nullValueCount = nullValueCounts.get(field.fieldId());
+ if (nullValueCount == null) {
+ return true;
+ }
+
+ return nullValueCount > 0;
+ }
+
+ static Builder builderFor(FileIO io, Iterable<ManifestFile> deleteManifests) {
+ return new Builder(io, Sets.newHashSet(deleteManifests));
+ }
+
+ // changed to public method.
+ public static Builder builderFor(Iterable<DeleteFile> deleteFiles) {
+ return new Builder(deleteFiles);
+ }
+
+ // changed to public class.
+ public static class Builder {
+ private final FileIO io;
+ private final Set<ManifestFile> deleteManifests;
+ private final Iterable<DeleteFile> deleteFiles;
+ private long minSequenceNumber = 0L;
+ private Map<Integer, PartitionSpec> specsById = null;
+ private Expression dataFilter = Expressions.alwaysTrue();
+ private Expression partitionFilter = Expressions.alwaysTrue();
+ private PartitionSet partitionSet = null;
+ private boolean caseSensitive = true;
+ private ExecutorService executorService = null;
+ private ScanMetrics scanMetrics = ScanMetrics.noop();
+ private boolean ignoreResiduals = false;
+
+ Builder(FileIO io, Set<ManifestFile> deleteManifests) {
+ this.io = io;
+ this.deleteManifests = Sets.newHashSet(deleteManifests);
+ this.deleteFiles = null;
+ }
+
+ Builder(Iterable<DeleteFile> deleteFiles) {
+ this.io = null;
+ this.deleteManifests = null;
+ this.deleteFiles = deleteFiles;
+ }
+
+ Builder afterSequenceNumber(long seq) {
+ this.minSequenceNumber = seq;
+ return this;
+ }
+
+ public Builder specsById(Map<Integer, PartitionSpec> newSpecsById) {
+ this.specsById = newSpecsById;
+ return this;
+ }
+
+ Builder filterData(Expression newDataFilter) {
+ Preconditions.checkArgument(
+ deleteFiles == null, "Index constructed from files does not support data filters");
+ this.dataFilter = Expressions.and(dataFilter, newDataFilter);
+ return this;
+ }
+
+ Builder filterPartitions(Expression newPartitionFilter) {
+ Preconditions.checkArgument(
+ deleteFiles == null, "Index constructed from files does not support partition filters");
+ this.partitionFilter = Expressions.and(partitionFilter, newPartitionFilter);
+ return this;
+ }
+
+ Builder filterPartitions(PartitionSet newPartitionSet) {
+ Preconditions.checkArgument(
+ deleteFiles == null, "Index constructed from files does not support partition filters");
+ this.partitionSet = newPartitionSet;
+ return this;
+ }
+
+ public Builder caseSensitive(boolean newCaseSensitive) {
+ this.caseSensitive = newCaseSensitive;
+ return this;
+ }
+
+ Builder planWith(ExecutorService newExecutorService) {
+ this.executorService = newExecutorService;
+ return this;
+ }
+
+ Builder scanMetrics(ScanMetrics newScanMetrics) {
+ this.scanMetrics = newScanMetrics;
+ return this;
+ }
+
+ Builder ignoreResiduals() {
+ this.ignoreResiduals = true;
+ return this;
+ }
+
+ private Iterable<DeleteFile> filterDeleteFiles() {
+ return Iterables.filter(deleteFiles, file -> file.dataSequenceNumber() > minSequenceNumber);
+ }
+
+ private Collection<DeleteFile> loadDeleteFiles() {
+ // read all of the matching delete manifests in parallel and accumulate the matching files in
+ // a queue
+ Queue<DeleteFile> files = new ConcurrentLinkedQueue<>();
+ Tasks.foreach(deleteManifestReaders())
+ .stopOnFailure()
+ .throwFailureWhenFinished()
+ .executeWith(executorService)
+ .run(
+ deleteFile -> {
+ try (CloseableIterable<ManifestEntry<DeleteFile>> reader = deleteFile) {
+ for (ManifestEntry<DeleteFile> entry : reader) {
+ if (entry.dataSequenceNumber() > minSequenceNumber) {
+ // copy with stats for better filtering against data file stats
+ files.add(entry.file().copy());
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeIOException(e, "Failed to close");
+ }
+ });
+ return files;
+ }
+
+ public DeleteFileIndex build() {
+ Iterable<DeleteFile> files = deleteFiles != null ? filterDeleteFiles() : loadDeleteFiles();
+
+ EqualityDeletes globalDeletes = new EqualityDeletes();
+ PartitionMap<EqualityDeletes> eqDeletesByPartition = PartitionMap.create(specsById);
+ PartitionMap<PositionDeletes> posDeletesByPartition = PartitionMap.create(specsById);
+ Map<String, PositionDeletes> posDeletesByPath = Maps.newHashMap();
+ Map<String, DeleteFile> dvByPath = Maps.newHashMap();
+
+ for (DeleteFile file : files) {
+ switch (file.content()) {
+ case POSITION_DELETES:
+ if (ContentFileUtil.isDV(file)) {
+ add(dvByPath, file);
+ } else {
+ add(posDeletesByPath, posDeletesByPartition, file);
+ }
+ break;
+ case EQUALITY_DELETES:
+ add(globalDeletes, eqDeletesByPartition, file);
+ break;
+ default:
+ throw new UnsupportedOperationException("Unsupported content: " + file.content());
+ }
+ ScanMetricsUtil.indexedDeleteFile(scanMetrics, file);
+ }
+
+ return new DeleteFileIndex(
+ globalDeletes.isEmpty() ? null : globalDeletes,
+ eqDeletesByPartition.isEmpty() ? null : eqDeletesByPartition,
+ posDeletesByPartition.isEmpty() ? null : posDeletesByPartition,
+ posDeletesByPath.isEmpty() ? null : posDeletesByPath,
+ dvByPath.isEmpty() ? null : dvByPath);
+ }
+
+ private void add(Map<String, DeleteFile> dvByPath, DeleteFile dv) {
+ String path = dv.referencedDataFile();
+ DeleteFile existingDV = dvByPath.putIfAbsent(path, dv);
+ if (existingDV != null) {
+ throw new ValidationException(
+ "Can't index multiple DVs for %s: %s and %s",
+ path, ContentFileUtil.dvDesc(dv), ContentFileUtil.dvDesc(existingDV));
+ }
+ }
+
+ private void add(
+ Map<String, PositionDeletes> deletesByPath,
+ PartitionMap<PositionDeletes> deletesByPartition,
+ DeleteFile file) {
+ String path = ContentFileUtil.referencedDataFileLocation(file);
+
+ PositionDeletes deletes;
+ if (path != null) {
+ deletes = deletesByPath.computeIfAbsent(path, ignored -> new PositionDeletes());
+ } else {
+ int specId = file.specId();
+ StructLike partition = file.partition();
+ deletes = deletesByPartition.computeIfAbsent(specId, partition, PositionDeletes::new);
+ }
+
+ deletes.add(file);
+ }
+
+ private void add(
+ EqualityDeletes globalDeletes,
+ PartitionMap<EqualityDeletes> deletesByPartition,
+ DeleteFile file) {
+ PartitionSpec spec = specsById.get(file.specId());
+
+ EqualityDeletes deletes;
+ if (spec.isUnpartitioned()) {
+ deletes = globalDeletes;
+ } else {
+ int specId = spec.specId();
+ StructLike partition = file.partition();
+ deletes = deletesByPartition.computeIfAbsent(specId, partition, EqualityDeletes::new);
+ }
+
+ deletes.add(spec, file);
+ }
+
+ private Iterable<CloseableIterable<ManifestEntry<DeleteFile>>> deleteManifestReaders() {
+ Expression entryFilter = ignoreResiduals ? Expressions.alwaysTrue() : dataFilter;
+
+ LoadingCache<Integer, Expression> partExprCache =
+ specsById == null
+ ? null
+ : Caffeine.newBuilder()
+ .build(
+ specId -> {
+ PartitionSpec spec = specsById.get(specId);
+ return Projections.inclusive(spec, caseSensitive).project(dataFilter);
+ });
+
+ LoadingCache<Integer, ManifestEvaluator> evalCache =
+ specsById == null
+ ? null
+ : Caffeine.newBuilder()
+ .build(
+ specId -> {
+ PartitionSpec spec = specsById.get(specId);
+ return ManifestEvaluator.forPartitionFilter(
+ Expressions.and(partitionFilter, partExprCache.get(specId)),
+ spec,
+ caseSensitive);
+ });
+
+ CloseableIterable<ManifestFile> closeableDeleteManifests =
+ CloseableIterable.withNoopClose(deleteManifests);
+ CloseableIterable<ManifestFile> matchingManifests =
+ evalCache == null
+ ? closeableDeleteManifests
+ : CloseableIterable.filter(
+ scanMetrics.skippedDeleteManifests(),
+ closeableDeleteManifests,
+ manifest ->
+ manifest.content() == ManifestContent.DELETES
+ && (manifest.hasAddedFiles() || manifest.hasExistingFiles())
+ && evalCache.get(manifest.partitionSpecId()).eval(manifest));
+
+ matchingManifests =
+ CloseableIterable.count(scanMetrics.scannedDeleteManifests(), matchingManifests);
+ return Iterables.transform(
+ matchingManifests,
+ manifest ->
+ ManifestFiles.readDeleteManifest(manifest, io, specsById)
+ .filterRows(entryFilter)
+ .filterPartitions(
+ Expressions.and(
+ partitionFilter, partExprCache.get(manifest.partitionSpecId())))
+ .filterPartitions(partitionSet)
+ .caseSensitive(caseSensitive)
+ .scanMetrics(scanMetrics)
+ .liveEntries());
+ }
+ }
+
+ /**
+ * Finds an index in the sorted array of sequence numbers where the given sequence number should
+ * be inserted or is found.
+ *
+ * <p>If the sequence number is present in the array, this method returns the index of the first
+ * occurrence of the sequence number. If the sequence number is not present, the method returns
+ * the index where the sequence number would be inserted while maintaining the sorted order of the
+ * array. This returned index ranges from 0 (inclusive) to the length of the array (inclusive).
+ *
+ * <p>This method is used to determine the subset of delete files that apply to a given data file.
+ *
+ * @param seqs an array of sequence numbers sorted in ascending order
+ * @param seq the sequence number to search for
+ * @return the index of the first occurrence or the insertion point
+ */
+ private static int findStartIndex(long[] seqs, long seq) {
+ int pos = Arrays.binarySearch(seqs, seq);
+ int start;
+ if (pos < 0) {
+ // the sequence number was not found, where it would be inserted is -(pos + 1)
+ start = -(pos + 1);
+ } else {
+ // the sequence number was found, but may not be the first
+ // find the first delete file with the given sequence number by decrementing the position
+ start = pos;
+ while (start > 0 && seqs[start - 1] >= seq) {
+ start -= 1;
+ }
+ }
+
+ return start;
+ }
+
+ private static DeleteFile[] concat(DeleteFile[]... deletes) {
+ return ArrayUtil.concat(DeleteFile.class, deletes);
+ }
+
+ // a group of position delete files sorted by the sequence number they apply to
+ static class PositionDeletes {
+ private static final Comparator<DeleteFile> SEQ_COMPARATOR =
+ Comparator.comparingLong(DeleteFile::dataSequenceNumber);
+
+ // indexed state
+ private long[] seqs = null;
+ private DeleteFile[] files = null;
+
+ // a buffer that is used to hold files before indexing
+ private volatile List<DeleteFile> buffer = Lists.newArrayList();
+
+ public void add(DeleteFile file) {
+ Preconditions.checkState(buffer != null, "Can't add files upon indexing");
+ buffer.add(file);
+ }
+
+ public DeleteFile[] filter(long seq) {
+ indexIfNeeded();
+
+ int start = findStartIndex(seqs, seq);
+
+ if (start >= files.length) {
+ return EMPTY_DELETES;
+ }
+
+ if (start == 0) {
+ return files;
+ }
+
+ int matchingFilesCount = files.length - start;
+ DeleteFile[] matchingFiles = new DeleteFile[matchingFilesCount];
+ System.arraycopy(files, start, matchingFiles, 0, matchingFilesCount);
+ return matchingFiles;
+ }
+
+ public Iterable<DeleteFile> referencedDeleteFiles() {
+ indexIfNeeded();
+ return Arrays.asList(files);
+ }
+
+ public boolean isEmpty() {
+ indexIfNeeded();
+ return files.length == 0;
+ }
+
+ private void indexIfNeeded() {
+ if (buffer != null) {
+ synchronized (this) {
+ if (buffer != null) {
+ this.files = indexFiles(buffer);
+ this.seqs = indexSeqs(files);
+ this.buffer = null;
+ }
+ }
+ }
+ }
+
+ private static DeleteFile[] indexFiles(List<DeleteFile> list) {
+ DeleteFile[] array = list.toArray(EMPTY_DELETES);
+ Arrays.sort(array, SEQ_COMPARATOR);
+ return array;
+ }
+
+ private static long[] indexSeqs(DeleteFile[] files) {
+ long[] seqs = new long[files.length];
+
+ for (int index = 0; index < files.length; index++) {
+ seqs[index] = files[index].dataSequenceNumber();
+ }
+
+ return seqs;
+ }
+ }
+
+ // a group of equality delete files sorted by the sequence number they apply to
+ static class EqualityDeletes {
+ private static final Comparator<EqualityDeleteFile> SEQ_COMPARATOR =
+ Comparator.comparingLong(EqualityDeleteFile::applySequenceNumber);
+ private static final EqualityDeleteFile[] EMPTY_EQUALITY_DELETES = new EqualityDeleteFile[0];
+
+ // indexed state
+ private long[] seqs = null;
+ private EqualityDeleteFile[] files = null;
+
+ // a buffer that is used to hold files before indexing
+ private volatile List<EqualityDeleteFile> buffer = Lists.newArrayList();
+
+ public void add(PartitionSpec spec, DeleteFile file) {
+ Preconditions.checkState(buffer != null, "Can't add files upon indexing");
+ buffer.add(new EqualityDeleteFile(spec, file));
+ }
+
+ public DeleteFile[] filter(long seq, DataFile dataFile) {
+ indexIfNeeded();
+
+ int start = findStartIndex(seqs, seq);
+
+ if (start >= files.length) {
+ return EMPTY_DELETES;
+ }
+
+ List<DeleteFile> matchingFiles = Lists.newArrayList();
+
+ for (int index = start; index < files.length; index++) {
+ EqualityDeleteFile file = files[index];
+ if (canContainEqDeletesForFile(dataFile, file)) {
+ matchingFiles.add(file.wrapped());
+ }
+ }
+
+ return matchingFiles.toArray(EMPTY_DELETES);
+ }
+
+ public Iterable<DeleteFile> referencedDeleteFiles() {
+ indexIfNeeded();
+ return Iterables.transform(Arrays.asList(files), EqualityDeleteFile::wrapped);
+ }
+
+ public boolean isEmpty() {
+ indexIfNeeded();
+ return files.length == 0;
+ }
+
+ private void indexIfNeeded() {
+ if (buffer != null) {
+ synchronized (this) {
+ if (buffer != null) {
+ this.files = indexFiles(buffer);
+ this.seqs = indexSeqs(files);
+ this.buffer = null;
+ }
+ }
+ }
+ }
+
+ private static EqualityDeleteFile[] indexFiles(List<EqualityDeleteFile> list) {
+ EqualityDeleteFile[] array = list.toArray(EMPTY_EQUALITY_DELETES);
+ Arrays.sort(array, SEQ_COMPARATOR);
+ return array;
+ }
+
+ private static long[] indexSeqs(EqualityDeleteFile[] files) {
+ long[] seqs = new long[files.length];
+
+ for (int index = 0; index < files.length; index++) {
+ seqs[index] = files[index].applySequenceNumber();
+ }
+
+ return seqs;
+ }
+ }
+
+ // an equality delete file wrapper that caches the converted boundaries for faster boundary checks
+ // this class is not meant to be exposed beyond the delete file index
+ private static class EqualityDeleteFile {
+ private final PartitionSpec spec;
+ private final DeleteFile wrapped;
+ private final long applySequenceNumber;
+ private volatile List<Types.NestedField> equalityFields = null;
+ private volatile Map<Integer, Object> convertedLowerBounds = null;
+ private volatile Map<Integer, Object> convertedUpperBounds = null;
+
+ EqualityDeleteFile(PartitionSpec spec, DeleteFile file) {
+ this.spec = spec;
+ this.wrapped = file;
+ this.applySequenceNumber = wrapped.dataSequenceNumber() - 1;
+ }
+
+ public DeleteFile wrapped() {
+ return wrapped;
+ }
+
+ public long applySequenceNumber() {
+ return applySequenceNumber;
+ }
+
+ public List<Types.NestedField> equalityFields() {
+ if (equalityFields == null) {
+ synchronized (this) {
+ if (equalityFields == null) {
+ List<Types.NestedField> fields = Lists.newArrayList();
+ for (int id : wrapped.equalityFieldIds()) {
+ Types.NestedField field = spec.schema().findField(id);
+ fields.add(field);
+ }
+ this.equalityFields = fields;
+ }
+ }
+ }
+
+ return equalityFields;
+ }
+
+ public Map<Integer, Long> valueCounts() {
+ return wrapped.valueCounts();
+ }
+
+ public Map<Integer, Long> nullValueCounts() {
+ return wrapped.nullValueCounts();
+ }
+
+ public boolean hasLowerAndUpperBounds() {
+ return wrapped.lowerBounds() != null && wrapped.upperBounds() != null;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T lowerBound(int id) {
+ return (T) lowerBounds().get(id);
+ }
+
+ private Map<Integer, Object> lowerBounds() {
+ if (convertedLowerBounds == null) {
+ synchronized (this) {
+ if (convertedLowerBounds == null) {
+ this.convertedLowerBounds = convertBounds(wrapped.lowerBounds());
+ }
+ }
+ }
+
+ return convertedLowerBounds;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T upperBound(int id) {
+ return (T) upperBounds().get(id);
+ }
+
+ private Map<Integer, Object> upperBounds() {
+ if (convertedUpperBounds == null) {
+ synchronized (this) {
+ if (convertedUpperBounds == null) {
+ this.convertedUpperBounds = convertBounds(wrapped.upperBounds());
+ }
+ }
+ }
+
+ return convertedUpperBounds;
+ }
+
+ private Map<Integer, Object> convertBounds(Map<Integer, ByteBuffer> bounds) {
+ Map<Integer, Object> converted = Maps.newHashMap();
+
+ if (bounds != null) {
+ for (Types.NestedField field : equalityFields()) {
+ int id = field.fieldId();
+ Type type = spec.schema().findField(id).type();
+ if (type.isPrimitiveType()) {
+ ByteBuffer bound = bounds.get(id);
+ if (bound != null) {
+ converted.put(id, Conversions.fromByteBuffer(type, bound));
+ }
+ }
+ }
+ }
+
+ return converted;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/04: Optimize location for tpch1000 (#59218)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch tpc_preview4-external
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c7572648fb955873e24cadc5cbbdcd2bb816b758
Author: Socrates <su...@selectdb.com>
AuthorDate: Sat Dec 20 22:35:38 2025 +0800
Optimize location for tpch1000 (#59218)
---
.../org/apache/doris/common/util/LocationPath.java | 89 ++++++++++++++++++----
.../datasource/iceberg/source/IcebergScanNode.java | 87 ++++++++++++++++++++-
.../property/storage/S3PropertyUtils.java | 48 +++++++++++-
3 files changed, 205 insertions(+), 19 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index e4b9aa0b25c..cbe2b01d912 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -96,27 +96,25 @@ public class LocationPath {
}
private static String parseScheme(String finalLocation) {
- String scheme = "";
- String[] schemeSplit = finalLocation.split(SCHEME_DELIM);
- if (schemeSplit.length > 1) {
- scheme = schemeSplit[0];
- } else {
- schemeSplit = finalLocation.split(NONSTANDARD_SCHEME_DELIM);
- if (schemeSplit.length > 1) {
- scheme = schemeSplit[0];
- }
+ // Use indexOf instead of split for better performance
+ int schemeDelimIndex = finalLocation.indexOf(SCHEME_DELIM);
+ if (schemeDelimIndex > 0) {
+ return finalLocation.substring(0, schemeDelimIndex);
+ }
+
+ int nonstandardDelimIndex = finalLocation.indexOf(NONSTANDARD_SCHEME_DELIM);
+ if (nonstandardDelimIndex > 0) {
+ return finalLocation.substring(0, nonstandardDelimIndex);
}
// if not get scheme, need consider /path/to/local to no scheme
- if (scheme.isEmpty()) {
- try {
- Paths.get(finalLocation);
- } catch (InvalidPathException exception) {
- throw new IllegalArgumentException("Fail to parse scheme, invalid location: " + finalLocation);
- }
+ try {
+ Paths.get(finalLocation);
+ } catch (InvalidPathException exception) {
+ throw new IllegalArgumentException("Fail to parse scheme, invalid location: " + finalLocation);
}
- return scheme;
+ return "";
}
/**
@@ -201,6 +199,65 @@ public class LocationPath {
}
}
+ /**
+ * Ultra-fast factory method that directly constructs LocationPath without any parsing.
+ * This is used when the normalized location is already known (e.g., from prefix transformation).
+ *
+ * @param normalizedLocation the already-normalized location string
+ * @param schema pre-computed schema
+ * @param fsIdentifier pre-computed filesystem identifier
+ * @param storageProperties the storage properties (can be null)
+ * @return a new LocationPath instance
+ */
+ public static LocationPath ofDirect(String normalizedLocation,
+ String schema,
+ String fsIdentifier,
+ StorageProperties storageProperties) {
+ return new LocationPath(schema, normalizedLocation, fsIdentifier, storageProperties);
+ }
+
+ /**
+ * Fast factory method that reuses pre-computed schema and fsIdentifier.
+ * This is optimized for batch processing where many files share the same bucket/prefix.
+ *
+ * @param location the input URI location string
+ * @param storageProperties pre-computed storage properties for normalization
+ * @param cachedSchema pre-computed schema (can be null to compute)
+ * @param cachedFsIdPrefix pre-computed fsIdentifier prefix like "s3://" (can be null to compute)
+ * @return a new LocationPath instance
+ */
+ public static LocationPath ofWithCache(String location,
+ StorageProperties storageProperties,
+ String cachedSchema,
+ String cachedFsIdPrefix) {
+ try {
+ String normalizedLocation = storageProperties.validateAndNormalizeUri(location);
+
+ String fsIdentifier;
+ if (cachedFsIdPrefix != null && normalizedLocation.startsWith(cachedFsIdPrefix)) {
+ // Fast path: extract authority from normalized location without full URI parsing
+ int authorityStart = cachedFsIdPrefix.length();
+ int authorityEnd = normalizedLocation.indexOf('/', authorityStart);
+ if (authorityEnd == -1) {
+ authorityEnd = normalizedLocation.length();
+ }
+ String authority = normalizedLocation.substring(authorityStart, authorityEnd);
+ fsIdentifier = cachedFsIdPrefix + authority;
+ } else {
+ // Fallback to full URI parsing
+ String encodedLocation = encodedLocation(normalizedLocation);
+ URI uri = URI.create(encodedLocation);
+ fsIdentifier = Strings.nullToEmpty(uri.getScheme()) + "://"
+ + Strings.nullToEmpty(uri.getAuthority());
+ }
+
+ String schema = cachedSchema != null ? cachedSchema : extractScheme(location);
+ return new LocationPath(schema, normalizedLocation, fsIdentifier, storageProperties);
+ } catch (UserException e) {
+ throw new StoragePropertiesException("Failed to create LocationPath for location: " + location, e);
+ }
+ }
+
/**
* Extracts the URI scheme (e.g., "s3", "hdfs") from the location string.
*
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index 133ac067644..698a6a380f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -133,6 +133,17 @@ public class IcebergScanNode extends FileQueryScanNode {
private Map<String, String> backendStorageProperties;
private Boolean isBatchMode = null;
+ // Cached values for LocationPath creation optimization
+ // These are lazily initialized on first use to avoid parsing overhead for each file
+ private volatile StorageProperties cachedStorageProperties;
+ private volatile String cachedSchema;
+ private volatile String cachedFsIdPrefix;
+ private volatile boolean locationPathCacheInitialized = false;
+ // Cache for path prefix transformation to avoid repeated S3URI parsing
+ // Maps original path prefix (e.g., "https://bucket.s3.amazonaws.com/") to normalized prefix (e.g., "s3://bucket/")
+ private volatile String cachedOriginalPathPrefix;
+ private volatile String cachedNormalizedPathPrefix;
+ private volatile String cachedFsIdentifier;
// for test
@VisibleForTesting
@@ -547,9 +558,83 @@ public class IcebergScanNode extends FileQueryScanNode {
return TableScanUtil.splitFiles(CloseableIterable.withNoopClose(tasks), targetSplitSize);
}
+ /**
+ * Initialize cached values for LocationPath creation on first use.
+ * This avoids repeated StorageProperties lookup, scheme parsing, and S3URI regex parsing for each file.
+ */
+ private void initLocationPathCache(String samplePath) {
+ if (locationPathCacheInitialized) {
+ return;
+ }
+ synchronized (this) {
+ if (locationPathCacheInitialized) {
+ return;
+ }
+ try {
+ // Create a LocationPath using the full method to get all cached values
+ LocationPath sampleLocationPath = LocationPath.of(samplePath, storagePropertiesMap);
+ cachedStorageProperties = sampleLocationPath.getStorageProperties();
+ cachedSchema = sampleLocationPath.getSchema();
+ cachedFsIdentifier = sampleLocationPath.getFsIdentifier();
+
+ // Extract fsIdPrefix like "s3://" from fsIdentifier like "s3://bucket"
+ int schemeEnd = cachedFsIdentifier.indexOf("://");
+ if (schemeEnd > 0) {
+ cachedFsIdPrefix = cachedFsIdentifier.substring(0, schemeEnd + 3);
+ }
+
+ // Cache path prefix mapping for fast transformation
+ // This allows subsequent files to skip S3URI regex parsing entirely
+ String normalizedPath = sampleLocationPath.getNormalizedLocation();
+
+ // Find the common prefix by looking for the last '/' before the filename
+ int lastSlashInOriginal = samplePath.lastIndexOf('/');
+ int lastSlashInNormalized = normalizedPath.lastIndexOf('/');
+
+ if (lastSlashInOriginal > 0 && lastSlashInNormalized > 0) {
+ cachedOriginalPathPrefix = samplePath.substring(0, lastSlashInOriginal + 1);
+ cachedNormalizedPathPrefix = normalizedPath.substring(0, lastSlashInNormalized + 1);
+ }
+
+ locationPathCacheInitialized = true;
+ } catch (Exception e) {
+ // If caching fails, we'll fall back to the full method each time
+ LOG.warn("Failed to initialize LocationPath cache, will use full parsing", e);
+ locationPathCacheInitialized = true;
+ }
+ }
+ }
+
+ /**
+ * Create a LocationPath with cached values for better performance.
+ * Uses cached path prefix mapping to completely bypass S3URI regex parsing for most files.
+ * Falls back to full parsing if cache is not available or path doesn't match cached prefix.
+ */
+ private LocationPath createLocationPathWithCache(String path) {
+ // Initialize cache on first call
+ if (!locationPathCacheInitialized) {
+ initLocationPathCache(path);
+ }
+
+ // Fast path: if path starts with cached original prefix, directly transform without any parsing
+ if (cachedOriginalPathPrefix != null && path.startsWith(cachedOriginalPathPrefix)) {
+ // Transform: replace original prefix with normalized prefix
+ String normalizedPath = cachedNormalizedPathPrefix + path.substring(cachedOriginalPathPrefix.length());
+ return LocationPath.ofDirect(normalizedPath, cachedSchema, cachedFsIdentifier, cachedStorageProperties);
+ }
+
+ // Medium path: use cached StorageProperties but still need validateAndNormalizeUri
+ if (cachedStorageProperties != null) {
+ return LocationPath.ofWithCache(path, cachedStorageProperties, cachedSchema, cachedFsIdPrefix);
+ }
+
+ // Fallback to full parsing
+ return LocationPath.of(path, storagePropertiesMap);
+ }
+
private Split createIcebergSplit(FileScanTask fileScanTask) {
String originalPath = fileScanTask.file().path().toString();
- LocationPath locationPath = LocationPath.of(originalPath, storagePropertiesMap);
+ LocationPath locationPath = createLocationPathWithCache(originalPath);
IcebergSplit split = new IcebergSplit(
locationPath,
fileScanTask.start(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java
index 99064b4e2e2..71360fc4799 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java
@@ -33,6 +33,15 @@ import java.util.Optional;
public class S3PropertyUtils {
private static final Logger LOG = LogManager.getLogger(S3PropertyUtils.class);
+ private static final String SCHEME_DELIM = "://";
+ private static final String S3_SCHEME_PREFIX = "s3://";
+
+ // S3-compatible schemes that can be converted to s3:// with simple string replacement
+ // Format: scheme://bucket/key -> s3://bucket/key
+ private static final String[] SIMPLE_S3_COMPATIBLE_SCHEMES = {
+ "s3a", "s3n", "oss", "cos", "cosn", "obs", "bos", "gs"
+ };
+
/**
* Constructs the S3 endpoint from a given URI in the props map.
*
@@ -113,7 +122,8 @@ public class S3PropertyUtils {
/**
* Validates and normalizes the given path into a standard S3 URI.
- * If the input already starts with "s3://", it is returned as-is.
+ * If the input already starts with a known S3-compatible scheme (s3://, s3a://, oss://, etc.),
+ * it is returned as-is to avoid expensive regex parsing.
* Otherwise, it is parsed and converted into an S3-compatible URI format.
*
* @param path the raw S3-style path or full URI
@@ -132,16 +142,50 @@ public class S3PropertyUtils {
if (StringUtils.isBlank(path)) {
throw new StoragePropertiesException("path is null");
}
- if (path.startsWith("s3://")) {
+
+ // Fast path 1: s3:// paths are already in the normalized format expected by BE
+ if (path.startsWith(S3_SCHEME_PREFIX)) {
return path;
}
+ // Fast path 2: simple S3-compatible schemes (oss://, cos://, s3a://, etc.)
+ // can be converted with simple string replacement: scheme://bucket/key -> s3://bucket/key
+ String normalized = trySimpleSchemeConversion(path);
+ if (normalized != null) {
+ return normalized;
+ }
+
+ // Full parsing path: for HTTP URLs and other complex formats
boolean usePathStyle = Boolean.parseBoolean(stringUsePathStyle);
boolean forceParsingByStandardUri = Boolean.parseBoolean(stringForceParsingByStandardUri);
S3URI s3uri = S3URI.create(path, usePathStyle, forceParsingByStandardUri);
return "s3" + S3URI.SCHEME_DELIM + s3uri.getBucket() + S3URI.PATH_DELIM + s3uri.getKey();
}
+ /**
+ * Try to convert simple S3-compatible scheme URIs to s3:// format using string replacement.
+ * This avoids expensive regex parsing for common cases like oss://bucket/key, s3a://bucket/key, etc.
+ *
+ * @param path the input path
+ * @return converted s3:// path if successful, null if the path doesn't match simple pattern
+ */
+ private static String trySimpleSchemeConversion(String path) {
+ int delimIndex = path.indexOf(SCHEME_DELIM);
+ if (delimIndex <= 0) {
+ return null;
+ }
+
+ String scheme = path.substring(0, delimIndex).toLowerCase();
+ for (String compatibleScheme : SIMPLE_S3_COMPATIBLE_SCHEMES) {
+ if (compatibleScheme.equals(scheme)) {
+ // Simple conversion: replace scheme with "s3"
+ // e.g., "oss://bucket/key" -> "s3://bucket/key"
+ return S3_SCHEME_PREFIX + path.substring(delimIndex + SCHEME_DELIM.length());
+ }
+ }
+ return null;
+ }
+
/**
* Extracts and returns the raw URI string from the given props map.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org