You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/03/16 23:50:41 UTC
(spark-kubernetes-operator) branch main updated: [SPARK-56004] Upgrade `java-operator-sdk` to 5.3.0
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new a783d6c [SPARK-56004] Upgrade `java-operator-sdk` to 5.3.0
a783d6c is described below
commit a783d6c91f86dfcbd32707bd0865a6cfb50a6840
Author: Aaruna Godthi <aa...@gmail.com>
AuthorDate: Mon Mar 16 16:50:34 2026 -0700
[SPARK-56004] Upgrade `java-operator-sdk` to 5.3.0
### What changes were proposed in this pull request?
Upgrade `java-operator-sdk` from 5.2.3 to 5.3.0 and adapt internal code to its API changes:
- Rename JUnit module artifact from `operator-framework-junit-5` to `operator-framework-junit`
- Rename `Metrics` interface methods per JOSDK migration guide (`receivedEvent` → `eventReceived`, `reconcileCustomResource` → `reconciliationSubmitted`, etc.)
- Update `reconciliationFailed` and `reconciliationFinished` signatures to include `RetryInfo` parameter
- Remove `monitorSizeOf` (removed from `Metrics` interface)
- Update `ResourceAction` import (moved to `event.source` package)
### Why are the changes needed?
JOSDK 5.3.0 introduces breaking changes to its internal `Metrics` interface. This upgrade keeps the dependency current and requires corresponding adapter code updates within the operator.
### Does this PR introduce _any_ user-facing change?
No. All changes are internal to the operator's JOSDK integration layer (`OperatorJosdkMetrics`). The operator's CRD API, behavior, and configuration remain unchanged.
### How was this patch tested?
- `./gradlew clean build` passes (compilation, checkstyle, spotless, spotbugs, unit tests)
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 4.6)
Closes #551 from aaruna/josdk-5.3.0-upgrade.
Authored-by: Aaruna Godthi <aa...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
gradle/libs.versions.toml | 4 +-
.../metrics/source/OperatorJosdkMetrics.java | 50 +++++++---------------
.../metrics/source/OperatorJosdkMetricsTest.java | 14 +++---
3 files changed, 25 insertions(+), 43 deletions(-)
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 79b87e0..b5295de 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -18,7 +18,7 @@
fabric8 = "7.6.1"
lombok = "1.18.42"
netty = "4.2.10.Final"
-operator-sdk = "5.2.3"
+operator-sdk = "5.3.0"
dropwizard-metrics = "4.2.38"
spark = "4.1.1"
hadoop = "3.4.3"
@@ -56,7 +56,7 @@ log4j-slf4j2-impl = { group = "org.apache.logging.log4j", name = "log4j-slf4j2-i
log4j-layout-template-json = { group = "org.apache.logging.log4j", name = "log4j-layout-template-json", version.ref = "log4j" }
slf4j-api = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" }
operator-framework = { group = "io.javaoperatorsdk", name = "operator-framework", version.ref = "operator-sdk"}
-operator-framework-junit5 = { group = "io.javaoperatorsdk", name = "operator-framework-junit-5", version.ref = "operator-sdk"}
+operator-framework-junit5 = { group = "io.javaoperatorsdk", name = "operator-framework-junit", version.ref = "operator-sdk"}
spotbugs-annotations = { group = "com.github.spotbugs", name = "spotbugs-annotations", version.ref = "spotbugs-tool"}
metrics-core = { group = "io.dropwizard.metrics", name = "metrics-core", version.ref = "dropwizard-metrics"}
metrics-jvm = { group = "io.dropwizard.metrics", name = "metrics-jvm", version.ref = "dropwizard-metrics"}
diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java
index 627716b..3d7c73b 100644
--- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java
+++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
-import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.javaoperatorsdk.operator.api.monitoring.Metrics;
@@ -35,7 +34,7 @@ import io.javaoperatorsdk.operator.processing.Controller;
import io.javaoperatorsdk.operator.processing.GroupVersionKind;
import io.javaoperatorsdk.operator.processing.event.Event;
import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceAction;
+import io.javaoperatorsdk.operator.processing.event.source.ResourceAction;
import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent;
import lombok.extern.slf4j.Slf4j;
@@ -64,7 +63,6 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
private static final String RECONCILIATIONS = "reconciliations";
private static final String RECONCILIATIONS_EXECUTIONS = RECONCILIATIONS + ".executions";
private static final String RECONCILIATIONS_QUEUE_SIZE = RECONCILIATIONS + ".queue.size";
- private static final String SIZE = "size";
private final Clock clock;
@@ -112,7 +110,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
* @param metadata Additional metadata associated with the event.
*/
@Override
- public void receivedEvent(Event event, Map<String, Object> metadata) {
+ public void eventReceived(Event event, Map<String, Object> metadata) {
log.debug("received event {}, metadata {}", event, metadata);
if (event instanceof ResourceEvent) {
final ResourceAction action = ((ResourceEvent) event).getAction();
@@ -204,7 +202,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
* @param metadata Additional metadata.
*/
@Override
- public void reconcileCustomResource(
+ public void reconciliationSubmitted(
HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {
log.debug(
"Reconcile custom resource {}, with retryInfo {} metadata {}",
@@ -225,12 +223,16 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
* Records metrics when a reconciliation fails.
*
* @param resource The custom resource for which reconciliation failed.
+ * @param retryInfo Information about retries, if any.
* @param exception The exception that caused the failure.
* @param metadata Additional metadata.
*/
@Override
- public void failedReconciliation(
- HasMetadata resource, Exception exception, Map<String, Object> metadata) {
+ public void reconciliationFailed(
+ HasMetadata resource,
+ RetryInfo retryInfo,
+ Exception exception,
+ Map<String, Object> metadata) {
log.error(
"Failed reconciliation for resource {} with metadata {}", resource, exception, exception);
String metricsPrefix = getMetricNamePrefix(resource.getClass());
@@ -239,13 +241,15 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
}
/**
- * Records metrics when a reconciliation finishes successfully.
+ * Records metrics when a reconciliation finishes.
*
* @param resource The custom resource for which reconciliation finished.
+ * @param retryInfo Information about retries, if any.
* @param metadata Additional metadata.
*/
@Override
- public void finishedReconciliation(HasMetadata resource, Map<String, Object> metadata) {
+ public void reconciliationFinished(
+ HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {
log.debug("Finished reconciliation for resource {} with metadata {}", resource, metadata);
String metricsPrefix = getMetricNamePrefix(resource.getClass());
getCounter(metricsPrefix, RECONCILIATION, FINISHED).inc();
@@ -259,7 +263,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
* @param metadata Additional metadata.
*/
@Override
- public void cleanupDoneFor(ResourceID resourceID, Map<String, Object> metadata) {
+ public void cleanupDone(ResourceID resourceID, Map<String, Object> metadata) {
log.debug("Cleanup Done for resource {} with metadata {}", resourceID, metadata);
String metricsPrefix = getMetricNamePrefix(resourceID.getClass());
getCounter(metricsPrefix, RECONCILIATION, CLEANUP).inc();
@@ -268,28 +272,6 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
.ifPresent(ns -> getCounter(metricsPrefix, ns, RECONCILIATION, CLEANUP).inc());
}
- /**
- * Monitors the size of a map and exposes it as a Gauge metric.
- *
- * @param map The map to monitor.
- * @param name The name of the metric.
- * @param <T> The type of the map.
- * @return The monitored map.
- */
- @Override
- public <T extends Map<?, ?>> T monitorSizeOf(T map, String name) {
- log.debug("Monitor size for {}", name);
- Gauge<Integer> gauge =
- new Gauge<>() {
- @Override
- public Integer getValue() {
- return map.size();
- }
- };
- gauges.put(MetricRegistry.name(name, SIZE), gauge);
- return map;
- }
-
/**
* Records metrics when a reconciliation execution starts.
*
@@ -297,7 +279,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
* @param metadata Additional metadata.
*/
@Override
- public void reconciliationExecutionStarted(HasMetadata resource, Map<String, Object> metadata) {
+ public void reconciliationStarted(HasMetadata resource, Map<String, Object> metadata) {
log.debug("Reconciliation execution started");
String namespace = resource.getMetadata().getNamespace();
String metricsPrefix = getMetricNamePrefix(resource.getClass());
@@ -318,7 +300,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
* @param metadata Additional metadata.
*/
@Override
- public void reconciliationExecutionFinished(HasMetadata resource, Map<String, Object> metadata) {
+ public void reconciliationSucceeded(HasMetadata resource, Map<String, Object> metadata) {
log.debug("Reconciliation execution finished");
String namespace = resource.getMetadata().getNamespace();
String metricsPrefix = getMetricNamePrefix(resource.getClass());
diff --git a/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java b/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java
index ed55996..d91e42f 100644
--- a/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java
+++ b/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java
@@ -33,7 +33,7 @@ import io.javaoperatorsdk.operator.api.reconciler.Constants;
import io.javaoperatorsdk.operator.processing.GroupVersionKind;
import io.javaoperatorsdk.operator.processing.event.Event;
import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceAction;
+import io.javaoperatorsdk.operator.processing.event.source.ResourceAction;
import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -93,7 +93,7 @@ class OperatorJosdkMetricsTest {
@Test
void testReconciliationFinished() {
- operatorMetrics.finishedReconciliation(buildNamespacedResource(), metadata);
+ operatorMetrics.reconciliationFinished(buildNamespacedResource(), null, metadata);
Map<String, Metric> metrics = operatorMetrics.metricRegistry().getMetrics();
assertEquals(2, metrics.size());
assertTrue(metrics.containsKey("configmap.default.reconciliation.finished"));
@@ -101,22 +101,22 @@ class OperatorJosdkMetricsTest {
}
@Test
- void testReconciliationExecutionStartedAndFinished() {
- operatorMetrics.reconciliationExecutionStarted(buildNamespacedResource(), metadata);
+ void testReconciliationStartedAndSucceeded() {
+ operatorMetrics.reconciliationStarted(buildNamespacedResource(), metadata);
Map<String, Metric> metrics = operatorMetrics.metricRegistry().getMetrics();
assertEquals(2, metrics.size());
assertTrue(metrics.containsKey("configmap.test-controller-name.reconciliations.executions"));
assertTrue(
metrics.containsKey("configmap.default.test-controller-name.reconciliations.executions"));
- operatorMetrics.reconciliationExecutionFinished(buildNamespacedResource(), metadata);
+ operatorMetrics.reconciliationSucceeded(buildNamespacedResource(), metadata);
assertEquals(3, metrics.size());
assertTrue(metrics.containsKey("configmap.test-controller-name.reconciliations.queue.size"));
}
@Test
- void testReceivedEvent() {
+ void testEventReceived() {
Event event = new ResourceEvent(ResourceAction.ADDED, resourceId, buildNamespacedResource());
- operatorMetrics.receivedEvent(event, metadata);
+ operatorMetrics.eventReceived(event, metadata);
Map<String, Metric> metrics = operatorMetrics.metricRegistry().getMetrics();
assertEquals(2, metrics.size());
assertTrue(metrics.containsKey("sparkapplication.added.resource.event"));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org