You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mc...@apache.org on 2018/08/01 20:57:37 UTC
spark git commit: [SPARK-24960][K8S] explicitly expose ports on
driver container
Repository: spark
Updated Branches:
refs/heads/master 95a9d5e3a -> f5113ea8d
[SPARK-24960][K8S] explicitly expose ports on driver container
https://issues.apache.org/jira/browse/SPARK-24960
## What changes were proposed in this pull request?
Expose ports explicitly in the driver container. The driver Service created expects to reach the driver Pod at specific ports which before this change, were not explicitly exposed and would likely cause connection issues (see https://github.com/apache-spark-on-k8s/spark/issues/617).
This is a port of the original PR created in the now-deprecated Kubernetes fork: https://github.com/apache-spark-on-k8s/spark/pull/618
## How was this patch tested?
Failure in https://github.com/apache-spark-on-k8s/spark/issues/617 reproduced on Kubernetes 1.6.x and 1.8.x. Built the driver image with this patch and observed fixed https://github.com/apache-spark-on-k8s/spark/issues/617 on Kubernetes 1.6.x.
Author: Adelbert Chang <Ad...@target.com>
Closes #21884 from adelbertc/k8s-expose-driver-ports.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f5113ea8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f5113ea8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f5113ea8
Branch: refs/heads/master
Commit: f5113ea8d79de724ec1579bc81a7abb61e44eeef
Parents: 95a9d5e
Author: Adelbert Chang <Ad...@target.com>
Authored: Wed Aug 1 13:57:33 2018 -0700
Committer: mcheah <mc...@palantir.com>
Committed: Wed Aug 1 13:57:33 2018 -0700
----------------------------------------------------------------------
.../org/apache/spark/deploy/k8s/Constants.scala | 1 +
.../k8s/features/BasicDriverFeatureStep.scala | 22 ++++++++++++++++++++
.../features/BasicDriverFeatureStepSuite.scala | 18 +++++++++++++++-
3 files changed, 40 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f5113ea8/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index 5ecdd3a..f82cd7f 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -47,6 +47,7 @@ private[spark] object Constants {
val DEFAULT_BLOCKMANAGER_PORT = 7079
val DRIVER_PORT_NAME = "driver-rpc-port"
val BLOCK_MANAGER_PORT_NAME = "blockmanager"
+ val UI_PORT_NAME = "spark-ui"
// Environment Variables
val ENV_DRIVER_URL = "SPARK_DRIVER_URL"
http://git-wip-us.apache.org/repos/asf/spark/blob/f5113ea8/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index 7e67b51..575bc54 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -27,6 +27,7 @@ import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.deploy.k8s.submit._
import org.apache.spark.internal.config._
+import org.apache.spark.ui.SparkUI
private[spark] class BasicDriverFeatureStep(
conf: KubernetesConf[KubernetesDriverSpecificConf])
@@ -72,10 +73,31 @@ private[spark] class BasicDriverFeatureStep(
("cpu", new QuantityBuilder(false).withAmount(limitCores).build())
}
+ val driverPort = conf.sparkConf.getInt("spark.driver.port", DEFAULT_DRIVER_PORT)
+ val driverBlockManagerPort = conf.sparkConf.getInt(
+ DRIVER_BLOCK_MANAGER_PORT.key,
+ DEFAULT_BLOCKMANAGER_PORT
+ )
+ val driverUIPort = SparkUI.getUIPort(conf.sparkConf)
val driverContainer = new ContainerBuilder(pod.container)
.withName(DRIVER_CONTAINER_NAME)
.withImage(driverContainerImage)
.withImagePullPolicy(conf.imagePullPolicy())
+ .addNewPort()
+ .withName(DRIVER_PORT_NAME)
+ .withContainerPort(driverPort)
+ .withProtocol("TCP")
+ .endPort()
+ .addNewPort()
+ .withName(BLOCK_MANAGER_PORT_NAME)
+ .withContainerPort(driverBlockManagerPort)
+ .withProtocol("TCP")
+ .endPort()
+ .addNewPort()
+ .withName(UI_PORT_NAME)
+ .withContainerPort(driverUIPort)
+ .withProtocol("TCP")
+ .endPort()
.addAllToEnv(driverCustomEnvs.asJava)
.addNewEnv()
.withName(ENV_DRIVER_BIND_ADDRESS)
http://git-wip-us.apache.org/repos/asf/spark/blob/f5113ea8/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index 165f46a..d98e113 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features
import scala.collection.JavaConverters._
-import io.fabric8.kubernetes.api.model.LocalObjectReferenceBuilder
+import io.fabric8.kubernetes.api.model.{ContainerPort, ContainerPortBuilder, LocalObjectReferenceBuilder}
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverSpecificConf, SparkPod}
@@ -26,6 +26,7 @@ import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.deploy.k8s.submit.JavaMainAppResource
import org.apache.spark.deploy.k8s.submit.PythonMainAppResource
+import org.apache.spark.ui.SparkUI
class BasicDriverFeatureStepSuite extends SparkFunSuite {
@@ -87,6 +88,14 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
assert(configuredPod.container.getImage === "spark-driver:latest")
assert(configuredPod.container.getImagePullPolicy === CONTAINER_IMAGE_PULL_POLICY)
+ val expectedPortNames = Set(
+ containerPort(DRIVER_PORT_NAME, DEFAULT_DRIVER_PORT),
+ containerPort(BLOCK_MANAGER_PORT_NAME, DEFAULT_BLOCKMANAGER_PORT),
+ containerPort(UI_PORT_NAME, SparkUI.DEFAULT_PORT)
+ )
+ val foundPortNames = configuredPod.container.getPorts.asScala.toSet
+ assert(expectedPortNames === foundPortNames)
+
assert(configuredPod.container.getEnv.size === 3)
val envs = configuredPod.container
.getEnv
@@ -203,4 +212,11 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
"spark.files" -> "https://localhost:9000/file1.txt,/opt/spark/file2.txt")
assert(additionalProperties === expectedSparkConf)
}
+
+ def containerPort(name: String, portNumber: Int): ContainerPort =
+ new ContainerPortBuilder()
+ .withName(name)
+ .withContainerPort(portNumber)
+ .withProtocol("TCP")
+ .build()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org