You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2016/06/15 14:09:35 UTC
spark git commit: [SPARK-15046][YARN] Parse value of token renewal
interval correctly.
Repository: spark
Updated Branches:
refs/heads/master 0ee9fd9e5 -> 40eeef952
[SPARK-15046][YARN] Parse value of token renewal interval correctly.
Use the config variable definition both to set and parse the value,
avoiding issues with code expecting the value in a different format.
Tested by running spark-submit with --principal / --keytab.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #13669 from vanzin/SPARK-15046.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/40eeef95
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/40eeef95
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/40eeef95
Branch: refs/heads/master
Commit: 40eeef95256b0740d759d921f0385023f0b91666
Parents: 0ee9fd9
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Wed Jun 15 09:09:21 2016 -0500
Committer: Tom Graves <tg...@yahoo-inc.com>
Committed: Wed Jun 15 09:09:21 2016 -0500
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala | 4 ++--
.../main/scala/org/apache/spark/internal/config/package.scala | 7 +++++++
yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala | 5 -----
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/40eeef95/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index 7a5fc86..bb1793d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -41,6 +41,7 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdenti
import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
import org.apache.spark.util.Utils
/**
@@ -288,8 +289,7 @@ class SparkHadoopUtil extends Logging {
credentials: Credentials): Long = {
val now = System.currentTimeMillis()
- val renewalInterval =
- sparkConf.getLong("spark.yarn.token.renewal.interval", (24 hours).toMillis)
+ val renewalInterval = sparkConf.get(TOKEN_RENEWAL_INTERVAL).get
credentials.getAllTokens.asScala
.filter(_.getKind == DelegationTokenIdentifier.HDFS_DELEGATION_KIND)
http://git-wip-us.apache.org/repos/asf/spark/blob/40eeef95/core/src/main/scala/org/apache/spark/internal/config/package.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 2c1e0b7..05dd683 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -17,6 +17,8 @@
package org.apache.spark.internal
+import java.util.concurrent.TimeUnit
+
import org.apache.spark.launcher.SparkLauncher
import org.apache.spark.network.util.ByteUnit
@@ -80,6 +82,11 @@ package object config {
.doc("Name of the Kerberos principal.")
.stringConf.createOptional
+ private[spark] val TOKEN_RENEWAL_INTERVAL = ConfigBuilder("spark.yarn.token.renewal.interval")
+ .internal()
+ .timeConf(TimeUnit.MILLISECONDS)
+ .createOptional
+
private[spark] val EXECUTOR_INSTANCES = ConfigBuilder("spark.executor.instances")
.intConf
.createOptional
http://git-wip-us.apache.org/repos/asf/spark/blob/40eeef95/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
index c4dd320..ad2412e 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -243,11 +243,6 @@ package object config {
.toSequence
.createWithDefault(Nil)
- private[spark] val TOKEN_RENEWAL_INTERVAL = ConfigBuilder("spark.yarn.token.renewal.interval")
- .internal()
- .timeConf(TimeUnit.MILLISECONDS)
- .createOptional
-
/* Private configs. */
private[spark] val CREDENTIALS_FILE_PATH = ConfigBuilder("spark.yarn.credentials.file")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org