You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/01/08 21:01:51 UTC
spark git commit: [SPARK-22992][K8S] Remove assumption of the DNS
domain
Repository: spark
Updated Branches:
refs/heads/master 40b983c3b -> eed82a0b2
[SPARK-22992][K8S] Remove assumption of the DNS domain
## What changes were proposed in this pull request?
Remove the use of FQDN to access the driver because it assumes that it's set up in a DNS zone - `cluster.local` which is common but not ubiquitous
Note that we already access the in-cluster API server through `kubernetes.default.svc`, so, by extension, this should work as well.
The alternative is to introduce DNS zones for both of those addresses.
## How was this patch tested?
Unit tests
cc vanzin liyinan926 mridulm mccheah
Author: foxish <ra...@google.com>
Closes #20187 from foxish/cluster.local.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eed82a0b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eed82a0b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eed82a0b
Branch: refs/heads/master
Commit: eed82a0b211352215316ec70dc48aefc013ad0b2
Parents: 40b983c
Author: foxish <ra...@google.com>
Authored: Mon Jan 8 13:01:45 2018 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Mon Jan 8 13:01:45 2018 -0800
----------------------------------------------------------------------
.../deploy/k8s/submit/steps/DriverServiceBootstrapStep.scala | 2 +-
.../k8s/submit/steps/DriverServiceBootstrapStepSuite.scala | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/eed82a0b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStep.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStep.scala
index eb594e4..34af7cd 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStep.scala
@@ -83,7 +83,7 @@ private[spark] class DriverServiceBootstrapStep(
.build()
val namespace = sparkConf.get(KUBERNETES_NAMESPACE)
- val driverHostname = s"${driverService.getMetadata.getName}.$namespace.svc.cluster.local"
+ val driverHostname = s"${driverService.getMetadata.getName}.$namespace.svc"
val resolvedSparkConf = driverSpec.driverSparkConf.clone()
.set(DRIVER_HOST_KEY, driverHostname)
.set("spark.driver.port", driverPort.toString)
http://git-wip-us.apache.org/repos/asf/spark/blob/eed82a0b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStepSuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStepSuite.scala
index 006ce26..78c8c3b 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/DriverServiceBootstrapStepSuite.scala
@@ -85,7 +85,7 @@ class DriverServiceBootstrapStepSuite extends SparkFunSuite with BeforeAndAfter
val resolvedDriverSpec = configurationStep.configureDriver(baseDriverSpec)
val expectedServiceName = SHORT_RESOURCE_NAME_PREFIX +
DriverServiceBootstrapStep.DRIVER_SVC_POSTFIX
- val expectedHostName = s"$expectedServiceName.my-namespace.svc.cluster.local"
+ val expectedHostName = s"$expectedServiceName.my-namespace.svc"
verifySparkConfHostNames(resolvedDriverSpec.driverSparkConf, expectedHostName)
}
@@ -120,7 +120,7 @@ class DriverServiceBootstrapStepSuite extends SparkFunSuite with BeforeAndAfter
val driverService = resolvedDriverSpec.otherKubernetesResources.head.asInstanceOf[Service]
val expectedServiceName = s"spark-10000${DriverServiceBootstrapStep.DRIVER_SVC_POSTFIX}"
assert(driverService.getMetadata.getName === expectedServiceName)
- val expectedHostName = s"$expectedServiceName.my-namespace.svc.cluster.local"
+ val expectedHostName = s"$expectedServiceName.my-namespace.svc"
verifySparkConfHostNames(resolvedDriverSpec.driverSparkConf, expectedHostName)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org