You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/03/02 19:41:15 UTC
[spark] branch branch-3.2 updated: [SPARK-38048][K8S][TESTS] Add `IntegrationTestBackend.describePods` to support all K8s test backends
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 79a68be [SPARK-38048][K8S][TESTS] Add `IntegrationTestBackend.describePods` to support all K8s test backends
79a68be is described below
commit 79a68becae88034a806a7fb6025da1eb5638d596
Author: William Hyun <wi...@apache.org>
AuthorDate: Thu Jan 27 19:02:39 2022 -0800
[SPARK-38048][K8S][TESTS] Add `IntegrationTestBackend.describePods` to support all K8s test backends
### What changes were proposed in this pull request?
This PR aims to add `IntegrationTestBackend.describePods` to support all K8s test backends
### Why are the changes needed?
Currently the docker based K8s tests cannot get the pod information when it fails.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manually.
Closes #35344 from williamhyun/describePOD.
Authored-by: William Hyun <wi...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 4f755772349117599d444579f04ecafb30cff092)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 3 +--
.../deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala | 5 +++++
.../spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala | 4 ----
.../k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala | 4 ++++
4 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 630dd03..7b0da1a 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -37,7 +37,6 @@ import org.scalatest.time.{Minutes, Seconds, Span}
import org.apache.spark.SparkFunSuite
import org.apache.spark.deploy.k8s.integrationtest.TestConstants._
import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory}
-import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config._
@@ -76,7 +75,7 @@ class KubernetesSuite extends SparkFunSuite
protected override def logForFailedTest(): Unit = {
logInfo("\n\n===== EXTRA LOGS FOR THE FAILED TEST\n")
logInfo("BEGIN DESCRIBE PODS for application\n" +
- Minikube.describePods(s"spark-app-locator=$appLocator").mkString("\n"))
+ testBackend.describePods(s"spark-app-locator=$appLocator").mkString("\n"))
logInfo("END DESCRIBE PODS for the application")
val driverPodOption = kubernetesTestComponents.kubernetesClient
.pods()
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala
index 56ddae0..36c3b6a 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala
@@ -19,6 +19,7 @@ package org.apache.spark.deploy.k8s.integrationtest.backend
import io.fabric8.kubernetes.client.DefaultKubernetesClient
+import org.apache.spark.deploy.k8s.integrationtest.ProcessUtils
import org.apache.spark.deploy.k8s.integrationtest.TestConstants._
import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.KubeConfigBackend
import org.apache.spark.deploy.k8s.integrationtest.backend.docker.DockerForDesktopBackend
@@ -28,6 +29,10 @@ private[spark] trait IntegrationTestBackend {
def initialize(): Unit
def getKubernetesClient: DefaultKubernetesClient
def cleanUp(): Unit = {}
+ def describePods(labels: String): Seq[String] =
+ ProcessUtils.executeProcess(
+ Array("bash", "-c", s"kubectl describe pods --all-namespaces -l $labels"),
+ timeout = 60, dumpOutput = false).filter { !_.contains("https://github.com/kubernetes") }
}
private[spark] object IntegrationTestBackendFactory {
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
index 1ebc644..9f99ede 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
@@ -111,10 +111,6 @@ private[spark] object Minikube extends Logging {
def minikubeServiceAction(args: String*): String = {
executeMinikube(true, "service", args: _*).head
}
-
- def describePods(labels: String): Seq[String] =
- Minikube.executeMinikube(false, "kubectl", "--", "describe", "pods", "--all-namespaces",
- "-l", labels)
}
private[spark] object MinikubeStatus extends Enumeration {
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala
index f92977d..8c8f848 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala
@@ -40,4 +40,8 @@ private[spark] object MinikubeTestBackend extends IntegrationTestBackend {
override def getKubernetesClient: DefaultKubernetesClient = {
defaultClient
}
+
+ override def describePods(labels: String): Seq[String] =
+ Minikube.executeMinikube(false, "kubectl", "--", "describe", "pods", "--all-namespaces",
+ "-l", labels)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org