You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/03/16 23:50:41 UTC

(spark-kubernetes-operator) branch main updated: [SPARK-56004] Upgrade `java-operator-sdk` to 5.3.0

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git


The following commit(s) were added to refs/heads/main by this push:
     new a783d6c  [SPARK-56004] Upgrade `java-operator-sdk` to 5.3.0
a783d6c is described below

commit a783d6c91f86dfcbd32707bd0865a6cfb50a6840
Author: Aaruna Godthi <aa...@gmail.com>
AuthorDate: Mon Mar 16 16:50:34 2026 -0700

    [SPARK-56004] Upgrade `java-operator-sdk` to 5.3.0
    
    ### What changes were proposed in this pull request?
    
      Upgrade `java-operator-sdk` from 5.2.3 to 5.3.0 and adapt internal code to its API changes:
      - Rename JUnit module artifact from `operator-framework-junit-5` to `operator-framework-junit`
      - Rename `Metrics` interface methods per JOSDK migration guide (`receivedEvent` → `eventReceived`, `reconcileCustomResource` → `reconciliationSubmitted`, etc.)
      - Update `reconciliationFailed` and `reconciliationFinished` signatures to include `RetryInfo` parameter
      - Remove `monitorSizeOf` (removed from `Metrics` interface)
      - Update `ResourceAction` import (moved to `event.source` package)
    
      ### Why are the changes needed?
    
      JOSDK 5.3.0 introduces breaking changes to its internal `Metrics` interface. This upgrade keeps the dependency current and requires corresponding adapter code updates within the operator.
    
      ### Does this PR introduce _any_ user-facing change?
    
      No. All changes are internal to the operator's JOSDK integration layer (`OperatorJosdkMetrics`). The operator's CRD API, behavior, and configuration remain unchanged.
    
      ### How was this patch tested?
    
      - `./gradlew clean build` passes (compilation, checkstyle, spotless, spotbugs, unit tests)
    
      ### Was this patch authored or co-authored using generative AI tooling?
    
      Generated-by: Claude Code (Claude Opus 4.6)
    
    Closes #551 from aaruna/josdk-5.3.0-upgrade.
    
    Authored-by: Aaruna Godthi <aa...@gmail.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 gradle/libs.versions.toml                          |  4 +-
 .../metrics/source/OperatorJosdkMetrics.java       | 50 +++++++---------------
 .../metrics/source/OperatorJosdkMetricsTest.java   | 14 +++---
 3 files changed, 25 insertions(+), 43 deletions(-)

diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 79b87e0..b5295de 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -18,7 +18,7 @@
 fabric8 = "7.6.1"
 lombok = "1.18.42"
 netty = "4.2.10.Final"
-operator-sdk = "5.2.3"
+operator-sdk = "5.3.0"
 dropwizard-metrics = "4.2.38"
 spark = "4.1.1"
 hadoop = "3.4.3"
@@ -56,7 +56,7 @@ log4j-slf4j2-impl = { group = "org.apache.logging.log4j", name = "log4j-slf4j2-i
 log4j-layout-template-json = { group = "org.apache.logging.log4j", name = "log4j-layout-template-json", version.ref = "log4j" }
 slf4j-api = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" }
 operator-framework = { group = "io.javaoperatorsdk", name = "operator-framework", version.ref = "operator-sdk"}
-operator-framework-junit5 = { group = "io.javaoperatorsdk", name = "operator-framework-junit-5", version.ref = "operator-sdk"}
+operator-framework-junit5 = { group = "io.javaoperatorsdk", name = "operator-framework-junit", version.ref = "operator-sdk"}
 spotbugs-annotations = { group = "com.github.spotbugs", name = "spotbugs-annotations", version.ref = "spotbugs-tool"}
 metrics-core = { group = "io.dropwizard.metrics", name = "metrics-core", version.ref = "dropwizard-metrics"}
 metrics-jvm = { group = "io.dropwizard.metrics", name = "metrics-jvm", version.ref = "dropwizard-metrics"}
diff --git a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java
index 627716b..3d7c73b 100644
--- a/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java
+++ b/spark-operator/src/main/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetrics.java
@@ -25,7 +25,6 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 
-import com.codahale.metrics.Gauge;
 import com.codahale.metrics.MetricRegistry;
 import io.fabric8.kubernetes.api.model.HasMetadata;
 import io.javaoperatorsdk.operator.api.monitoring.Metrics;
@@ -35,7 +34,7 @@ import io.javaoperatorsdk.operator.processing.Controller;
 import io.javaoperatorsdk.operator.processing.GroupVersionKind;
 import io.javaoperatorsdk.operator.processing.event.Event;
 import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceAction;
+import io.javaoperatorsdk.operator.processing.event.source.ResourceAction;
 import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent;
 import lombok.extern.slf4j.Slf4j;
 
@@ -64,7 +63,6 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
   private static final String RECONCILIATIONS = "reconciliations";
   private static final String RECONCILIATIONS_EXECUTIONS = RECONCILIATIONS + ".executions";
   private static final String RECONCILIATIONS_QUEUE_SIZE = RECONCILIATIONS + ".queue.size";
-  private static final String SIZE = "size";
 
   private final Clock clock;
 
@@ -112,7 +110,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
    * @param metadata Additional metadata associated with the event.
    */
   @Override
-  public void receivedEvent(Event event, Map<String, Object> metadata) {
+  public void eventReceived(Event event, Map<String, Object> metadata) {
     log.debug("received event {}, metadata {}", event, metadata);
     if (event instanceof ResourceEvent) {
       final ResourceAction action = ((ResourceEvent) event).getAction();
@@ -204,7 +202,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
    * @param metadata Additional metadata.
    */
   @Override
-  public void reconcileCustomResource(
+  public void reconciliationSubmitted(
       HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {
     log.debug(
         "Reconcile custom resource {}, with retryInfo {} metadata {}",
@@ -225,12 +223,16 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
    * Records metrics when a reconciliation fails.
    *
    * @param resource The custom resource for which reconciliation failed.
+   * @param retryInfo Information about retries, if any.
    * @param exception The exception that caused the failure.
    * @param metadata Additional metadata.
    */
   @Override
-  public void failedReconciliation(
-      HasMetadata resource, Exception exception, Map<String, Object> metadata) {
+  public void reconciliationFailed(
+      HasMetadata resource,
+      RetryInfo retryInfo,
+      Exception exception,
+      Map<String, Object> metadata) {
     log.error(
         "Failed reconciliation for resource {} with metadata {}", resource, exception, exception);
     String metricsPrefix = getMetricNamePrefix(resource.getClass());
@@ -239,13 +241,15 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
   }
 
   /**
-   * Records metrics when a reconciliation finishes successfully.
+   * Records metrics when a reconciliation finishes.
    *
    * @param resource The custom resource for which reconciliation finished.
+   * @param retryInfo Information about retries, if any.
    * @param metadata Additional metadata.
    */
   @Override
-  public void finishedReconciliation(HasMetadata resource, Map<String, Object> metadata) {
+  public void reconciliationFinished(
+      HasMetadata resource, RetryInfo retryInfo, Map<String, Object> metadata) {
     log.debug("Finished reconciliation for resource {} with metadata {}", resource, metadata);
     String metricsPrefix = getMetricNamePrefix(resource.getClass());
     getCounter(metricsPrefix, RECONCILIATION, FINISHED).inc();
@@ -259,7 +263,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
    * @param metadata Additional metadata.
    */
   @Override
-  public void cleanupDoneFor(ResourceID resourceID, Map<String, Object> metadata) {
+  public void cleanupDone(ResourceID resourceID, Map<String, Object> metadata) {
     log.debug("Cleanup Done for resource {} with metadata {}", resourceID, metadata);
     String metricsPrefix = getMetricNamePrefix(resourceID.getClass());
     getCounter(metricsPrefix, RECONCILIATION, CLEANUP).inc();
@@ -268,28 +272,6 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
         .ifPresent(ns -> getCounter(metricsPrefix, ns, RECONCILIATION, CLEANUP).inc());
   }
 
-  /**
-   * Monitors the size of a map and exposes it as a Gauge metric.
-   *
-   * @param map The map to monitor.
-   * @param name The name of the metric.
-   * @param <T> The type of the map.
-   * @return The monitored map.
-   */
-  @Override
-  public <T extends Map<?, ?>> T monitorSizeOf(T map, String name) {
-    log.debug("Monitor size for {}", name);
-    Gauge<Integer> gauge =
-        new Gauge<>() {
-          @Override
-          public Integer getValue() {
-            return map.size();
-          }
-        };
-    gauges.put(MetricRegistry.name(name, SIZE), gauge);
-    return map;
-  }
-
   /**
    * Records metrics when a reconciliation execution starts.
    *
@@ -297,7 +279,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
    * @param metadata Additional metadata.
    */
   @Override
-  public void reconciliationExecutionStarted(HasMetadata resource, Map<String, Object> metadata) {
+  public void reconciliationStarted(HasMetadata resource, Map<String, Object> metadata) {
     log.debug("Reconciliation execution started");
     String namespace = resource.getMetadata().getNamespace();
     String metricsPrefix = getMetricNamePrefix(resource.getClass());
@@ -318,7 +300,7 @@ public class OperatorJosdkMetrics extends BaseOperatorSource implements Source,
    * @param metadata Additional metadata.
    */
   @Override
-  public void reconciliationExecutionFinished(HasMetadata resource, Map<String, Object> metadata) {
+  public void reconciliationSucceeded(HasMetadata resource, Map<String, Object> metadata) {
     log.debug("Reconciliation execution finished");
     String namespace = resource.getMetadata().getNamespace();
     String metricsPrefix = getMetricNamePrefix(resource.getClass());
diff --git a/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java b/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java
index ed55996..d91e42f 100644
--- a/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java
+++ b/spark-operator/src/test/java/org/apache/spark/k8s/operator/metrics/source/OperatorJosdkMetricsTest.java
@@ -33,7 +33,7 @@ import io.javaoperatorsdk.operator.api.reconciler.Constants;
 import io.javaoperatorsdk.operator.processing.GroupVersionKind;
 import io.javaoperatorsdk.operator.processing.event.Event;
 import io.javaoperatorsdk.operator.processing.event.ResourceID;
-import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceAction;
+import io.javaoperatorsdk.operator.processing.event.source.ResourceAction;
 import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -93,7 +93,7 @@ class OperatorJosdkMetricsTest {
 
   @Test
   void testReconciliationFinished() {
-    operatorMetrics.finishedReconciliation(buildNamespacedResource(), metadata);
+    operatorMetrics.reconciliationFinished(buildNamespacedResource(), null, metadata);
     Map<String, Metric> metrics = operatorMetrics.metricRegistry().getMetrics();
     assertEquals(2, metrics.size());
     assertTrue(metrics.containsKey("configmap.default.reconciliation.finished"));
@@ -101,22 +101,22 @@ class OperatorJosdkMetricsTest {
   }
 
   @Test
-  void testReconciliationExecutionStartedAndFinished() {
-    operatorMetrics.reconciliationExecutionStarted(buildNamespacedResource(), metadata);
+  void testReconciliationStartedAndSucceeded() {
+    operatorMetrics.reconciliationStarted(buildNamespacedResource(), metadata);
     Map<String, Metric> metrics = operatorMetrics.metricRegistry().getMetrics();
     assertEquals(2, metrics.size());
     assertTrue(metrics.containsKey("configmap.test-controller-name.reconciliations.executions"));
     assertTrue(
         metrics.containsKey("configmap.default.test-controller-name.reconciliations.executions"));
-    operatorMetrics.reconciliationExecutionFinished(buildNamespacedResource(), metadata);
+    operatorMetrics.reconciliationSucceeded(buildNamespacedResource(), metadata);
     assertEquals(3, metrics.size());
     assertTrue(metrics.containsKey("configmap.test-controller-name.reconciliations.queue.size"));
   }
 
   @Test
-  void testReceivedEvent() {
+  void testEventReceived() {
     Event event = new ResourceEvent(ResourceAction.ADDED, resourceId, buildNamespacedResource());
-    operatorMetrics.receivedEvent(event, metadata);
+    operatorMetrics.eventReceived(event, metadata);
     Map<String, Metric> metrics = operatorMetrics.metricRegistry().getMetrics();
     assertEquals(2, metrics.size());
     assertTrue(metrics.containsKey("sparkapplication.added.resource.event"));


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org