You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2022/04/26 07:16:16 UTC

[GitHub] [spark] martin-g commented on a diff in pull request #36333: [SPARK-39006][K8S] Check PVC claimName must be OnDemand when multiple executors required

martin-g commented on code in PR #36333:
URL: https://github.com/apache/spark/pull/36333#discussion_r858360310


##########
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala:
##########
@@ -120,6 +122,16 @@ private[spark] class MountVolumesFeatureStep(conf: KubernetesConf)
   override def getAdditionalKubernetesResources(): Seq[HasMetadata] = {
     additionalResources.toSeq
   }
+
+  private def checkPVCOnDemandWhenMultiExecutors(claimName: String): Unit = {
+    val executorInstances = conf.get(EXECUTOR_INSTANCES)
+    if (executorInstances.isEmpty) return
+    if (claimName != PVC_ON_DEMAND && executorInstances.get > 1) {
+      throw new IllegalArgumentException("ClaimName of PVC must be " +
+        PVC_ON_DEMAND +
+        " when multiple executors required")

Review Comment:
   ```suggestion
           " when multiple executors are required")
   ```



##########
resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala:
##########
@@ -148,6 +149,23 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
     assert(executorPVC.getClaimName.endsWith("-exec-1-pvc-0"))
   }
 
+  test("SPARK-39006 PVC claimName must be onDemand when multiple executors") {
+    val volumeConf = KubernetesVolumeSpec(
+      "testVolume",
+      "/tmp",
+      "",
+      mountReadOnly = true,
+      KubernetesPVCVolumeConf("testClaimName")
+    )
+    val conf = new SparkConf().set(EXECUTOR_INSTANCES, 2)
+    val executorConf =
+      KubernetesTestConf.createExecutorConf(sparkConf = conf, volumes = Seq(volumeConf))
+    val executorStep = new MountVolumesFeatureStep(executorConf)
+    assertThrows[IllegalArgumentException] {

Review Comment:
   IMO it would be good to assert on the exception message too. Or use a specialized exception.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org