You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2022/03/05 07:25:13 UTC
[GitHub] [spark] martin-g commented on a change in pull request #35733: [SPARK-38188][K8S][TEST][FOLLOWUP] Cleanup pod resource after queue test
martin-g commented on a change in pull request #35733:
URL: https://github.com/apache/spark/pull/35733#discussion_r820060556
##########
File path: resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
##########
@@ -173,19 +205,20 @@ private[spark] trait VolcanoTestsSuite { k8sSuite: KubernetesSuite =>
}
test("SPARK-38188: Run SparkPi jobs with 2 queues (all enable)", k8sTestTag, volcanoTag) {
Review comment:
```suggestion
test("SPARK-38188: Run SparkPi jobs with 2 queues (all enabled)", k8sTestTag, volcanoTag) {
```
##########
File path: resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
##########
@@ -29,21 +29,52 @@ import scala.concurrent.Future
import io.fabric8.kubernetes.api.model.Pod
import io.fabric8.kubernetes.client.NamespacedKubernetesClient
import io.fabric8.volcano.client.VolcanoClient
+import org.scalatest.BeforeAndAfterEach
import org.scalatest.concurrent.Eventually
import org.apache.spark.SparkFunSuite
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.features.VolcanoFeatureStep
import org.apache.spark.internal.config.NETWORK_AUTH_ENABLED
-private[spark] trait VolcanoTestsSuite { k8sSuite: KubernetesSuite =>
+private[spark] trait VolcanoTestsSuite extends BeforeAndAfterEach { k8sSuite: KubernetesSuite =>
import VolcanoTestsSuite._
import org.apache.spark.deploy.k8s.integrationtest.VolcanoSuite.volcanoTag
import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{k8sTestTag, INTERVAL, TIMEOUT}
lazy val volcanoClient: VolcanoClient
= kubernetesTestComponents.kubernetesClient.adapt(classOf[VolcanoClient])
lazy val k8sClient: NamespacedKubernetesClient = kubernetesTestComponents.kubernetesClient
+ protected var testGroups: mutable.Set[String] = _
+
+ private def deletePodInTestGroup(): Unit = {
+ testGroups.map{ g =>
+ kubernetesTestComponents.kubernetesClient.pods().withLabel("spark-group-locator", g).delete()
+ Eventually.eventually(TIMEOUT, INTERVAL) {
+ val size = kubernetesTestComponents.kubernetesClient
+ .pods()
+ .withLabel("spark-app-locator", g)
+ .list().getItems.size()
+ assert(size === 0)
+ }
+ }
+ }
+
+ override protected def beforeEach(): Unit = {
Review comment:
it is better to call `super.beforeEach()` here even if it does nothing
##########
File path: resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
##########
@@ -29,21 +29,52 @@ import scala.concurrent.Future
import io.fabric8.kubernetes.api.model.Pod
import io.fabric8.kubernetes.client.NamespacedKubernetesClient
import io.fabric8.volcano.client.VolcanoClient
+import org.scalatest.BeforeAndAfterEach
import org.scalatest.concurrent.Eventually
import org.apache.spark.SparkFunSuite
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.features.VolcanoFeatureStep
import org.apache.spark.internal.config.NETWORK_AUTH_ENABLED
-private[spark] trait VolcanoTestsSuite { k8sSuite: KubernetesSuite =>
+private[spark] trait VolcanoTestsSuite extends BeforeAndAfterEach { k8sSuite: KubernetesSuite =>
import VolcanoTestsSuite._
import org.apache.spark.deploy.k8s.integrationtest.VolcanoSuite.volcanoTag
import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{k8sTestTag, INTERVAL, TIMEOUT}
lazy val volcanoClient: VolcanoClient
= kubernetesTestComponents.kubernetesClient.adapt(classOf[VolcanoClient])
lazy val k8sClient: NamespacedKubernetesClient = kubernetesTestComponents.kubernetesClient
+ protected var testGroups: mutable.Set[String] = _
+
+ private def deletePodInTestGroup(): Unit = {
+ testGroups.map{ g =>
+ kubernetesTestComponents.kubernetesClient.pods().withLabel("spark-group-locator", g).delete()
+ Eventually.eventually(TIMEOUT, INTERVAL) {
+ val size = kubernetesTestComponents.kubernetesClient
+ .pods()
+ .withLabel("spark-app-locator", g)
+ .list().getItems.size()
+ assert(size === 0)
+ }
+ }
+ }
+
+ override protected def beforeEach(): Unit = {
+ testGroups = mutable.Set.empty
+ }
+
+ override protected def afterEach(): Unit = {
+ super.afterEach()
+ deletePodInTestGroup()
Review comment:
usually it is better to cleanup this instance's resources and then ask the parent to cleanup its resources. The reverse of what `beforeEach()` does
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org