You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by yu...@apache.org on 2024/10/21 16:46:09 UTC
(incubator-gluten) branch main updated: [GLUTEN-7585][VL] Fix S3 and GCS configs (#7586)
This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ec6e6f7803 [GLUTEN-7585][VL] Fix S3 and GCS configs (#7586)
ec6e6f7803 is described below
commit ec6e6f7803d5ad9e8f235201569bcad5465627f8
Author: Deepak Majeti <de...@ibm.com>
AuthorDate: Mon Oct 21 12:46:04 2024 -0400
[GLUTEN-7585][VL] Fix S3 and GCS configs (#7586)
The S3 endpoint default needs to be removed. We should use the library default.
Remove empty defaults for S3 configs from the Backend config as they are unnecessary.
Pass all GCS configs to the Backend config.
Fixes: #7585
---
.../scala/org/apache/gluten/GlutenConfig.scala | 35 +++++++++-------------
1 file changed, 14 insertions(+), 21 deletions(-)
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index 12b6128519..41431f1a11 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -520,13 +520,17 @@ object GlutenConfig {
// GCS config
val GCS_PREFIX = "fs.gs."
- val GCS_STORAGE_ROOT_URL = "fs.gs.storage.root.url"
- val SPARK_GCS_STORAGE_ROOT_URL: String = HADOOP_PREFIX + GCS_STORAGE_ROOT_URL
- val GCS_AUTH_TYPE = "fs.gs.auth.type"
- val SPARK_GCS_AUTH_TYPE: String = HADOOP_PREFIX + GCS_AUTH_TYPE
- val GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE = "fs.gs.auth.service.account.json.keyfile"
+ val STORAGE_ROOT_URL = "storage.root.url"
+ val AUTH_TYPE = "auth.type"
+ val AUTH_SERVICE_ACCOUNT_JSON_KEYFILE = "auth.service.account.json.keyfile"
+ val HTTP_MAX_RETRY_COUNT = "http.max.retry"
+ val HTTP_MAX_RETRY_TIME = "http.max.retry-time"
+ val SPARK_GCS_STORAGE_ROOT_URL: String = HADOOP_PREFIX + GCS_PREFIX + STORAGE_ROOT_URL
+ val SPARK_GCS_AUTH_TYPE: String = HADOOP_PREFIX + GCS_PREFIX + AUTH_TYPE
val SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE: String =
- HADOOP_PREFIX + GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE
+ HADOOP_PREFIX + GCS_PREFIX + AUTH_SERVICE_ACCOUNT_JSON_KEYFILE
+ val SPARK_GCS_MAX_RETRY_COUNT: String = HADOOP_PREFIX + GCS_PREFIX + HTTP_MAX_RETRY_COUNT
+ val SPARK_GCS_MAX_RETRY_TIME: String = HADOOP_PREFIX + GCS_PREFIX + HTTP_MAX_RETRY_TIME
// QAT config
val GLUTEN_QAT_BACKEND_NAME = "qat"
@@ -738,14 +742,9 @@ object GlutenConfig {
// some configs having default values
val keyWithDefault = ImmutableList.of(
- (SPARK_S3_ACCESS_KEY, ""),
- (SPARK_S3_SECRET_KEY, ""),
- (SPARK_S3_ENDPOINT, "localhost:9000"),
(SPARK_S3_CONNECTION_SSL_ENABLED, "false"),
(SPARK_S3_PATH_STYLE_ACCESS, "true"),
(SPARK_S3_USE_INSTANCE_CREDENTIALS, "false"),
- (SPARK_S3_IAM, ""),
- (SPARK_S3_IAM_SESSION_NAME, ""),
(SPARK_S3_RETRY_MAX_ATTEMPTS, "20"),
(SPARK_S3_CONNECTION_MAXIMUM, "15"),
(AWS_S3_CONNECT_TIMEOUT.key, AWS_S3_CONNECT_TIMEOUT.defaultValueString),
@@ -799,16 +798,10 @@ object GlutenConfig {
.filter(_._1.startsWith(SPARK_ABFS_ACCOUNT_KEY))
.foreach(entry => nativeConfMap.put(entry._1, entry._2))
- // handle GCS config
- if (conf.contains(SPARK_GCS_AUTH_TYPE)) {
- nativeConfMap.put(SPARK_GCS_AUTH_TYPE, conf(SPARK_GCS_AUTH_TYPE))
- }
-
- if (conf.contains(SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE)) {
- nativeConfMap.put(
- SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE,
- conf(SPARK_GCS_AUTH_SERVICE_ACCOUNT_JSON_KEYFILE))
- }
+ // put in all GCS configs
+ conf
+ .filter(_._1.startsWith(HADOOP_PREFIX + GCS_PREFIX))
+ .foreach(entry => nativeConfMap.put(entry._1, entry._2))
// return
nativeConfMap
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org