You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/03/30 21:09:28 UTC
spark git commit: [SPARK-23640][CORE] Fix hadoop config may override
spark config
Repository: spark
Updated Branches:
refs/heads/master bc8d09311 -> ae9172017
[SPARK-23640][CORE] Fix hadoop config may override spark config
## What changes were proposed in this pull request?
It may be get `spark.shuffle.service.port` from https://github.com/apache/spark/blob/9745ec3a61c99be59ef6a9d5eebd445e8af65b7a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala#L459
Therefore, the client configuration `spark.shuffle.service.port` does not working unless the configuration is `spark.hadoop.spark.shuffle.service.port`.
- This configuration is not working:
```
bin/spark-sql --master yarn --conf spark.shuffle.service.port=7338
```
- This configuration works:
```
bin/spark-sql --master yarn --conf spark.hadoop.spark.shuffle.service.port=7338
```
This PR fix this issue.
## How was this patch tested?
It's difficult to carry out unit testing. But I've tested it manually.
Author: Yuming Wang <yu...@ebay.com>
Closes #20785 from wangyum/SPARK-23640.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae917201
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae917201
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae917201
Branch: refs/heads/master
Commit: ae9172017c361e5c1039bc2ca94048117021974a
Parents: bc8d093
Author: Yuming Wang <yu...@ebay.com>
Authored: Fri Mar 30 14:09:14 2018 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Fri Mar 30 14:09:14 2018 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/util/Utils.scala | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ae917201/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 5caedeb..d2be932 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2302,16 +2302,20 @@ private[spark] object Utils extends Logging {
}
/**
- * Return the value of a config either through the SparkConf or the Hadoop configuration
- * if this is Yarn mode. In the latter case, this defaults to the value set through SparkConf
- * if the key is not set in the Hadoop configuration.
+ * Return the value of a config either through the SparkConf or the Hadoop configuration.
+ * We Check whether the key is set in the SparkConf before look at any Hadoop configuration.
+ * If the key is set in SparkConf, no matter whether it is running on YARN or not,
+ * gets the value from SparkConf.
+ * Only when the key is not set in SparkConf and running on YARN,
+ * gets the value from Hadoop configuration.
*/
def getSparkOrYarnConfig(conf: SparkConf, key: String, default: String): String = {
- val sparkValue = conf.get(key, default)
- if (conf.get(SparkLauncher.SPARK_MASTER, null) == "yarn") {
- new YarnConfiguration(SparkHadoopUtil.get.newConfiguration(conf)).get(key, sparkValue)
+ if (conf.contains(key)) {
+ conf.get(key, default)
+ } else if (conf.get(SparkLauncher.SPARK_MASTER, null) == "yarn") {
+ new YarnConfiguration(SparkHadoopUtil.get.newConfiguration(conf)).get(key, default)
} else {
- sparkValue
+ default
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org