You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/29 13:17:06 UTC
(spark) branch branch-4.2 updated: [SPARK-59842][K8S] Fix `spark.kubernetes.executor.resizeInterval` default value and `0` handling
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new fc01a9e3165f [SPARK-59842][K8S] Fix `spark.kubernetes.executor.resizeInterval` default value and `0` handling
fc01a9e3165f is described below
commit fc01a9e3165f9c371074390f4b3e6f7aeabef04b
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Tue Sep 29 06:07:51 2026 -0700
[SPARK-59842][K8S] Fix `spark.kubernetes.executor.resizeInterval` default value and `0` handling
### What changes were proposed in this pull request?
This PR aims to fix `spark.kubernetes.executor.resizeInterval` handling in `ExecutorResizePlugin`.
- Change the default value from `0s` to `1m` to match the existing runtime behavior.
- Disable `ExecutorResizePlugin` when the interval is `0`, like `ExecutorPVCResizePlugin` does.
### Why are the changes needed?
Since Apache Spark 4.2.0, `spark.kubernetes.executor.resizeInterval` is documented as `0s` (disabled) by default, but `ExecutorResizePlugin` actually runs every minute when the config is unset. Moreover, setting `0` as documented makes `SparkContext` fail to start due to `IllegalArgumentException` from `scheduleAtFixedRate`.
### Does this PR introduce _any_ user-facing change?
Yes, but this is a bug fix where `spark.kubernetes.executor.resizeInterval=0` disables `ExecutorResizePlugin` instead of failing `SparkContext` initialization. The behavior of the unset case is unchanged.
### How was this patch tested?
Pass the CIs with the newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #59114 from dongjoon-hyun/SPARK-59842.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 8582186d9d52477029a41729d7e47a1da3c5b3f0)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
docs/running-on-kubernetes.md | 4 +--
.../scala/org/apache/spark/deploy/k8s/Config.scala | 4 +--
.../cluster/k8s/ExecutorResizePlugin.scala | 9 ++++--
.../cluster/k8s/ExecutorResizePluginSuite.scala | 32 ++++++++++++++++++++++
4 files changed, 42 insertions(+), 7 deletions(-)
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 0478b62e750a..dc4c0b37033f 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -1804,9 +1804,9 @@ See the [configuration page](configuration.html) for information on Spark config
</tr>
<tr>
<td><code>spark.kubernetes.executor.resizeInterval</code></td>
- <td><code>0s</code></td>
+ <td><code>1min</code></td>
<td>
- Interval between executor resize operations. To disable, set 0 (default).
+ Interval between executor resize operations. To disable, set 0.
Takes effect only when <code>org.apache.spark.scheduler.cluster.k8s.ExecutorResizePlugin</code>
is registered via <code>spark.plugins</code>.
</td>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 2cff728cff9e..71bfa2b37e38 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -240,11 +240,11 @@ private[spark] object Config extends Logging {
val EXECUTOR_RESIZE_INTERVAL =
ConfigBuilder("spark.kubernetes.executor.resizeInterval")
- .doc("Interval between executor resize operations. To disable, set 0 (default)")
+ .doc("Interval between executor resize operations. To disable, set 0.")
.version("4.2.0")
.timeConf(TimeUnit.SECONDS)
.checkValue(_ >= 0, "Interval should be non-negative")
- .createWithDefault(0)
+ .createWithDefaultString("1m")
val EXECUTOR_RESIZE_THRESHOLD =
ConfigBuilder("spark.kubernetes.executor.resizeThreshold")
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
index ad4611b52fc7..0b9e33672827 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePlugin.scala
@@ -32,7 +32,7 @@ import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{CLASS_NAME, CONFIG, CONFIG2, EXECUTOR_ID, MEMORY_SIZE}
-import org.apache.spark.util.{ThreadUtils, Utils}
+import org.apache.spark.util.ThreadUtils
/**
* Spark plugin to monitor executor pod memory usage and increase the memory limit
@@ -59,8 +59,11 @@ class ExecutorResizeDriverPlugin extends DriverPlugin with Logging {
return Map.empty[String, String].asJava
}
- val interval = Utils.timeStringAsSeconds(
- sc.conf.get(EXECUTOR_RESIZE_INTERVAL.key, "1m"))
+ val interval = sc.conf.get(EXECUTOR_RESIZE_INTERVAL)
+ if (interval <= 0) {
+ logInfo("ExecutorResizePlugin disabled (interval <= 0).")
+ return Map.empty[String, String].asJava
+ }
val threshold = sc.conf.get(EXECUTOR_RESIZE_THRESHOLD)
val factor = sc.conf.get(EXECUTOR_RESIZE_FACTOR)
val namespace = sc.conf.get(KUBERNETES_NAMESPACE)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
index e07d93ca107c..cfa4b0fd780a 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorResizePluginSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.scheduler.cluster.k8s
import java.util.Collections
+import java.util.concurrent.ScheduledThreadPoolExecutor
import scala.jdk.CollectionConverters._
@@ -334,4 +335,35 @@ class ExecutorResizePluginSuite
}
}
}
+
+ test("SPARK-59842: resizeInterval defaults to 1 minute") {
+ assert(new SparkConf(false).get(EXECUTOR_RESIZE_INTERVAL) === 60)
+ }
+
+ test("SPARK-59842: init returns early when resizeInterval is 0") {
+ val plugin = new ExecutorResizeDriverPlugin()
+ val sparkConf = new SparkConf().set(EXECUTOR_RESIZE_INTERVAL.key, "0")
+ val sc = mock(classOf[SparkContext])
+ when(sc.conf).thenReturn(sparkConf)
+ val pluginCtx = mock(classOf[PluginContext])
+
+ val result = plugin.init(sc, pluginCtx)
+
+ assert(result.isEmpty)
+ }
+
+ test("SPARK-59842: init schedules the resize task by default") {
+ val plugin = new ExecutorResizeDriverPlugin()
+ val sc = mock(classOf[SparkContext])
+ when(sc.conf).thenReturn(new SparkConf())
+ val pluginCtx = mock(classOf[PluginContext])
+ try {
+ assert(plugin.init(sc, pluginCtx).isEmpty)
+ val field = plugin.getClass.getDeclaredField("periodicService")
+ field.setAccessible(true)
+ assert(field.get(plugin).asInstanceOf[ScheduledThreadPoolExecutor].getQueue.size === 1)
+ } finally {
+ plugin.shutdown()
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org