You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/12 14:53:22 UTC
[doris] 01/14: [Fix](multi-catalog) delete hdfs hedged configs at BE side. (#25094)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b380ea37b6f969fad8c3d4d8d47e7216c3bb1d56
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Wed Oct 11 23:25:30 2023 +0800
[Fix](multi-catalog) delete hdfs hedged configs at BE side. (#25094)
Issue Number: close #25093
We can set hdfs hedged configs when creating catalog, just like this:
```
CREATE CATALOG `test_ctl` PROPERTIES (
...
"dfs.client.hedged.read.threadpool.size" = "128",
"dfs.client.hedged.read.threshold.millis" = "500",
...
);
```
It is redundant to set these configs at BE side, and it will brings an occasional bug at #25093 .
---
be/src/common/config.cpp | 4 ----
be/src/common/config.h | 10 ----------
be/src/io/hdfs_builder.cpp | 11 -----------
3 files changed, 25 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index c299e2a8e3e..ec96cdc4f97 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1062,10 +1062,6 @@ DEFINE_mInt64(lookup_connection_cache_bytes_limit, "4294967296");
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
DEFINE_mInt64(LZ4_HC_compression_level, "9");
-DEFINE_Bool(enable_hdfs_hedged_read, "false");
-DEFINE_Int32(hdfs_hedged_read_thread_num, "128");
-DEFINE_Int32(hdfs_hedged_read_threshold_time, "500");
-
DEFINE_mBool(enable_merge_on_write_correctness_check, "true");
// The secure path with user files, used in the `local` table function.
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 443f2773057..311aad13643 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1113,16 +1113,6 @@ DECLARE_mInt64(lookup_connection_cache_bytes_limit);
// level of compression when using LZ4_HC, whose defalut value is LZ4HC_CLEVEL_DEFAULT
DECLARE_mInt64(LZ4_HC_compression_level);
-// whether to enable hdfs hedged read.
-// If set to true, it will be enabled even if user not enable it when creating catalog
-DECLARE_Bool(enable_hdfs_hedged_read);
-// hdfs hedged read thread pool size, for "dfs.client.hedged.read.threadpool.size"
-// Maybe overwritten by the value specified when creating catalog
-DECLARE_Int32(hdfs_hedged_read_thread_num);
-// the threshold of doing hedged read, for "dfs.client.hedged.read.threshold.millis"
-// Maybe overwritten by the value specified when creating catalog
-DECLARE_Int32(hdfs_hedged_read_threshold_time);
-
DECLARE_mBool(enable_merge_on_write_correctness_check);
// The secure path with user files, used in the `local` table function.
diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp
index 754eb61680b..41cac611de7 100644
--- a/be/src/io/hdfs_builder.cpp
+++ b/be/src/io/hdfs_builder.cpp
@@ -146,17 +146,6 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_
}
}
-#ifdef USE_HADOOP_HDFS
- if (config::enable_hdfs_hedged_read) {
- hdfsBuilderConfSetStr(builder->get(), "dfs.client.hedged.read.threadpool.size",
- std::to_string(config::hdfs_hedged_read_thread_num).c_str());
- hdfsBuilderConfSetStr(builder->get(), "dfs.client.hedged.read.threshold.millis",
- std::to_string(config::hdfs_hedged_read_threshold_time).c_str());
- LOG(INFO) << "set hdfs hedged read config: " << config::hdfs_hedged_read_thread_num << ", "
- << config::hdfs_hedged_read_threshold_time;
- }
-#endif
-
hdfsBuilderConfSetStr(builder->get(), "ipc.client.fallback-to-simple-auth-allowed", "true");
if (builder->is_need_kinit()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org