You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/14 05:08:36 UTC
(spark-kubernetes-operator) branch main updated: [SPARK-59475] Add `suspend` field to BaseSpec for SparkApplication and SparkCluster
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new e936807 [SPARK-59475] Add `suspend` field to BaseSpec for SparkApplication and SparkCluster
e936807 is described below
commit e936807a8a31f1ce3c2b5a53a8c548298df26e10
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Sep 13 22:08:29 2026 -0700
[SPARK-59475] Add `suspend` field to BaseSpec for SparkApplication and SparkCluster
### What changes were proposed in this pull request?
This PR adds a `suspend` field to `BaseSpec` with a default value of `false`. Since both `ApplicationSpec` and `ClusterSpec` extend `BaseSpec`, the `spec.suspend` field is now opened on both `SparkApplication` and `SparkCluster` custom resources.
### Why are the changes needed?
Adding `spec.suspend` to `BaseSpec` provides a common field across `SparkApplication` and `SparkCluster` custom resources to support workload pausing and resumption. Following the common workload convention in the Kubernetes ecosystem (also widely used by batch queueing systems such as [Kueue](https://kueue.sigs.k8s.io/)), opening this field lays the foundation for suspension semantics and integration with external queue/lifecycle controllers.
### Does this PR introduce _any_ user-facing change?
No behavior change. This only adds an optional `spec.suspend` field (boolean, default: `false`) to both `SparkApplication` and `SparkCluster` CRDs:
```yaml
spec:
suspend: false
```
### How was this patch tested?
- Added and ran unit tests:
- `BaseSpecTest`
- `ApplicationSpecTest`
- `ClusterSpecTest`
- Verified CRD schema consistency with `./gradlew :spark-operator-api:assertGeneratedCRDMatchesHelmChart`
- Validated Helm charts with `helm lint --strict build-tools/helm/spark-kubernetes-operator`
- Verified full quality gate with `./gradlew build`
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Antigravity
Closes #821 from dongjoon-hyun/SPARK-59475.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../sparkapplications.spark.apache.org-v1.yaml | 3 ++
.../crds/sparkclusters.spark.apache.org-v1.yaml | 3 ++
.../apache/spark/k8s/operator/spec/BaseSpec.java | 4 ++
.../k8s/operator/spec/ApplicationSpecTest.java | 1 +
.../spark/k8s/operator/spec/BaseSpecTest.java | 62 ++++++++++++++++++++++
.../spark/k8s/operator/spec/ClusterSpecTest.java | 2 +
6 files changed, 75 insertions(+)
diff --git a/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml
index 1579369..b46b8e7 100644
--- a/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml
+++ b/build-tools/helm/spark-kubernetes-operator/crds/sparkapplications.spark.apache.org-v1.yaml
@@ -16512,6 +16512,9 @@ spec:
type: object
sparkRFiles:
type: string
+ suspend:
+ default: false
+ type: boolean
required:
- runtimeVersions
type: object
diff --git a/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml b/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml
index c4b7cac..72c5668 100644
--- a/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml
+++ b/build-tools/helm/spark-kubernetes-operator/crds/sparkclusters.spark.apache.org-v1.yaml
@@ -10913,6 +10913,9 @@ spec:
additionalProperties:
type: string
type: object
+ suspend:
+ default: false
+ type: boolean
workerSpec:
properties:
horizontalPodAutoscalerSpec:
diff --git a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java
index e9d3770..535710c 100644
--- a/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java
+++ b/spark-operator-api/src/main/java/org/apache/spark/k8s/operator/spec/BaseSpec.java
@@ -24,6 +24,7 @@ import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
+import io.fabric8.generator.annotation.Default;
import lombok.Data;
import org.apache.spark.k8s.operator.diff.Diffable;
@@ -37,5 +38,8 @@ import org.apache.spark.k8s.operator.diff.Diffable;
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class BaseSpec implements Diffable<BaseSpec> {
+ @Default("false")
+ protected boolean suspend = false;
+
protected Map<String, String> sparkConf = new HashMap<>();
}
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ApplicationSpecTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ApplicationSpecTest.java
index 2706ee5..9dce6f0 100644
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ApplicationSpecTest.java
+++ b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ApplicationSpecTest.java
@@ -39,6 +39,7 @@ class ApplicationSpecTest {
@Test
void testInitSpecWithDefaults() {
ApplicationSpec spec = new ApplicationSpec();
+ assertFalse(spec.isSuspend());
assertEquals(ClusterMode, spec.getDeploymentMode());
assertNull(spec.getDriverSpec());
assertNull(spec.getExecutorSpec());
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/BaseSpecTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/BaseSpecTest.java
new file mode 100644
index 0000000..5ac34c3
--- /dev/null
+++ b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/BaseSpecTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.spark.k8s.operator.spec;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+class BaseSpecTest {
+ @Test
+ void testDefaults() {
+ BaseSpec spec = new BaseSpec();
+ assertFalse(spec.isSuspend());
+ assertNotNull(spec.getSparkConf());
+ assertTrue(spec.getSparkConf().isEmpty());
+ }
+
+ @Test
+ void testSettersAndGetters() {
+ BaseSpec spec = new BaseSpec();
+ spec.setSuspend(true);
+ assertTrue(spec.isSuspend());
+ spec.setSuspend(false);
+ assertFalse(spec.isSuspend());
+ }
+
+ @Test
+ void testSerialization() throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+ BaseSpec spec = new BaseSpec();
+ spec.setSuspend(true);
+ String json = mapper.writeValueAsString(spec);
+ BaseSpec deserialized = mapper.readValue(json, BaseSpec.class);
+ assertEquals(spec, deserialized);
+ assertTrue(deserialized.isSuspend());
+
+ String emptyJson = "{}";
+ BaseSpec fromEmpty = mapper.readValue(emptyJson, BaseSpec.class);
+ assertFalse(fromEmpty.isSuspend());
+ }
+}
diff --git a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ClusterSpecTest.java b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ClusterSpecTest.java
index 5a125bd..7f06147 100644
--- a/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ClusterSpecTest.java
+++ b/spark-operator-api/src/test/java/org/apache/spark/k8s/operator/spec/ClusterSpecTest.java
@@ -20,6 +20,7 @@
package org.apache.spark.k8s.operator.spec;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import org.junit.jupiter.api.Test;
@@ -35,6 +36,7 @@ class ClusterSpecTest {
@Test
void testInitSpecWithDefaults() {
ClusterSpec spec1 = new ClusterSpec();
+ assertFalse(spec1.isSuspend());
assertNull(spec1.runtimeVersions.jdkVersion);
assertNull(spec1.runtimeVersions.scalaVersion);
assertNull(spec1.runtimeVersions.sparkVersion);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org