[spark] branch master updated: [SPARK-38503][K8S] Warn if `VolcanoFeatureStep.getAdditionalPreKubernetesResources` is used in executor side

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new b65c8a106e2 [SPARK-38503][K8S] Warn if `VolcanoFeatureStep.getAdditionalPreKubernetesResources` is used in executor side
b65c8a106e2 is described below

commit b65c8a106e289410397deea1cd8837c277f68598
Author: Yikun Jiang <yi...@gmail.com>
AuthorDate: Wed Aug 10 10:40:37 2022 -0700

    [SPARK-38503][K8S] Warn if `VolcanoFeatureStep.getAdditionalPreKubernetesResources` is used in executor side
    
    ### What changes were proposed in this pull request?
    - Add warn for getAdditionalPreKubernetesResources in executor side
    
    ### Why are the changes needed?
    Only `spark.kubernetes.scheduler.volcano.podGroupTemplateFile` is going to be supported in v3.3, and it only be called when building driver side feature steps.
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    - K8S IT passed
    - CI passed
    
    Closes #35786 from Yikun/SPARK-38455-FOLLOWUP.
    
    Authored-by: Yikun Jiang <yi...@gmail.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala | 8 +++++++-
 .../spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala       | 7 +++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
index 091923a78ef..ab84393dd5d 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
@@ -21,9 +21,10 @@ import io.fabric8.volcano.client.DefaultVolcanoClient
 import io.fabric8.volcano.scheduling.v1beta1.{PodGroup, PodGroupSpec}
 
 import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverConf, KubernetesExecutorConf, SparkPod}
+import org.apache.spark.internal.Logging
 
 private[spark] class VolcanoFeatureStep extends KubernetesDriverCustomFeatureConfigStep
-  with KubernetesExecutorCustomFeatureConfigStep {
+  with KubernetesExecutorCustomFeatureConfigStep with Logging {
   import VolcanoFeatureStep._
 
   private var kubernetesConf: KubernetesConf = _
@@ -40,6 +41,11 @@ private[spark] class VolcanoFeatureStep extends KubernetesDriverCustomFeatureCon
   }
 
   override def getAdditionalPreKubernetesResources(): Seq[HasMetadata] = {
+    if (kubernetesConf.isInstanceOf[KubernetesExecutorConf]) {
+      logWarning("VolcanoFeatureStep#getAdditionalPreKubernetesResources() is not supported " +
+        "for executor.")
+      return Seq.empty
+    }
     val client = new DefaultVolcanoClient
     val template = kubernetesConf.getOption(POD_GROUP_TEMPLATE_FILE_KEY)
     val pg = template.map(client.podGroups.load(_).get).getOrElse(new PodGroup())
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
index d0d1f5ee5e1..dab414e0e19 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
@@ -66,4 +66,11 @@ class VolcanoFeatureStepSuite extends SparkFunSuite {
     assert(podGroup.getSpec.getPriorityClassName == "driver-priority")
     assert(podGroup.getSpec.getQueue == "driver-queue")
   }
+
+  test("SPARK-38503: return empty for executor pre resource") {
+    val kubernetesConf = KubernetesTestConf.createExecutorConf(new SparkConf())
+    val step = new VolcanoFeatureStep()
+    step.init(kubernetesConf)
+    assert(step.getAdditionalPreKubernetesResources() === Seq.empty)
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org