You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by wa...@apache.org on 2022/05/19 13:21:06 UTC

[flink-kubernetes-operator] branch release-1.0 updated: [FLINK-27675] Improve manual savepoint tracking

This is an automated email from the ASF dual-hosted git repository.

wangyang0918 pushed a commit to branch release-1.0
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git


The following commit(s) were added to refs/heads/release-1.0 by this push:
     new 851b905  [FLINK-27675] Improve manual savepoint tracking
851b905 is described below

commit 851b9059055c849bd4fd2d0c4293b6774a911354
Author: Matyas Orhidi <ma...@apple.com>
AuthorDate: Wed May 18 14:22:02 2022 +0200

    [FLINK-27675] Improve manual savepoint tracking
---
 .../operator/crd/status/SavepointInfo.java         |   5 +-
 .../operator/observer/SavepointFetchResult.java    |   8 +-
 .../operator/observer/SavepointObserver.java       |  59 +++++++-----
 .../deployment/AbstractDeploymentObserver.java     |  14 +++
 .../kubernetes/operator/service/FlinkService.java  |  18 +++-
 .../kubernetes/operator/TestingFlinkService.java   |  22 ++++-
 .../deployment/ApplicationObserverTest.java        | 103 +++++++++++++++++++--
 .../sessionjob/SessionJobObserverTest.java         |  20 +++-
 8 files changed, 199 insertions(+), 50 deletions(-)

diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java
index b20691d..a2eb133 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java
@@ -23,6 +23,7 @@ import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -51,9 +52,11 @@ public class SavepointInfo {
         this.triggerTimestamp = System.currentTimeMillis();
     }
 
-    public void resetTrigger() {
+    public boolean resetTrigger() {
+        boolean reseted = StringUtils.isNotEmpty(this.triggerId);
         this.triggerId = "";
         this.triggerTimestamp = 0L;
+        return reseted;
     }
 
     public void updateLastSavepoint(Savepoint savepoint) {
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java
index 74524c0..9f111fa 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java
@@ -25,13 +25,9 @@ import lombok.Value;
 @Value
 public class SavepointFetchResult {
     Savepoint savepoint;
-    boolean isTriggered;
+    boolean pending;
     String error;
 
-    public static SavepointFetchResult notTriggered() {
-        return new SavepointFetchResult(null, false, null);
-    }
-
     public static SavepointFetchResult error(String error) {
         return new SavepointFetchResult(null, false, error);
     }
@@ -41,6 +37,6 @@ public class SavepointFetchResult {
     }
 
     public static SavepointFetchResult completed(Savepoint savepoint) {
-        return new SavepointFetchResult(savepoint, true, null);
+        return new SavepointFetchResult(savepoint, false, null);
     }
 }
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java
index daa8142..91726d7 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java
@@ -27,8 +27,8 @@ import org.apache.flink.kubernetes.operator.crd.status.CommonStatus;
 import org.apache.flink.kubernetes.operator.crd.status.Savepoint;
 import org.apache.flink.kubernetes.operator.crd.status.SavepointInfo;
 import org.apache.flink.kubernetes.operator.exception.ReconciliationException;
-import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
 import org.apache.flink.kubernetes.operator.service.FlinkService;
+import org.apache.flink.kubernetes.operator.utils.EventUtils;
 import org.apache.flink.kubernetes.operator.utils.SavepointUtils;
 import org.apache.flink.kubernetes.operator.utils.StatusHelper;
 
@@ -70,7 +70,18 @@ public class SavepointObserver<STATUS extends CommonStatus<?>> {
                         .orElse(null);
 
         observeTriggeredSavepointProgress(savepointInfo, jobId, deployedConfig)
-                .ifPresent(err -> ReconciliationUtils.updateForReconciliationError(resource, err));
+                .ifPresent(
+                        err ->
+                                EventUtils.createOrUpdateEvent(
+                                        flinkService.getKubernetesClient(),
+                                        resource,
+                                        EventUtils.Type.Warning,
+                                        "SavepointError",
+                                        "Savepoint failed for savepointTriggerNonce: "
+                                                + resource.getSpec()
+                                                        .getJob()
+                                                        .getSavepointTriggerNonce(),
+                                        EventUtils.Component.Operator));
 
         // We only need to observe latest checkpoint/savepoint for terminal jobs
         if (JobStatus.valueOf(jobStatus.getState()).isGloballyTerminalState()) {
@@ -106,36 +117,36 @@ public class SavepointObserver<STATUS extends CommonStatus<?>> {
             LOG.debug("Savepoint not in progress");
             return Optional.empty();
         }
-        LOG.info("Observing savepoint status");
-        SavepointFetchResult savepointFetchResult;
-        try {
-            savepointFetchResult =
-                    flinkService.fetchSavepointInfo(
-                            currentSavepointInfo.getTriggerId(), jobID, deployedConfig);
-        } catch (Exception e) {
-            LOG.error("Exception while fetching savepoint info", e);
-            return Optional.empty();
-        }
-
-        if (!savepointFetchResult.isTriggered()) {
-            String error = savepointFetchResult.getError();
-            if (error != null
-                    || SavepointUtils.gracePeriodEnded(
-                            configManager.getOperatorConfiguration(), currentSavepointInfo)) {
-                String errorMsg = error != null ? error : "Savepoint status unknown";
-                LOG.error(errorMsg);
+        LOG.info("Observing savepoint status.");
+        SavepointFetchResult savepointFetchResult =
+                flinkService.fetchSavepointInfo(
+                        currentSavepointInfo.getTriggerId(), jobID, deployedConfig);
+
+        if (savepointFetchResult.isPending()) {
+            if (SavepointUtils.gracePeriodEnded(
+                    configManager.getOperatorConfiguration(), currentSavepointInfo)) {
+                String errorMsg =
+                        "Savepoint operation timed out after "
+                                + configManager
+                                        .getOperatorConfiguration()
+                                        .getSavepointTriggerGracePeriod();
                 currentSavepointInfo.resetTrigger();
+                LOG.error(errorMsg);
                 return Optional.of(errorMsg);
+            } else {
+                LOG.info("Savepoint operation not finished yet, waiting within grace period...");
+                return Optional.empty();
             }
-            LOG.info("Savepoint operation not running, waiting within grace period...");
         }
-        if (savepointFetchResult.getSavepoint() == null) {
-            LOG.info("Savepoint is still in progress...");
-            return Optional.empty();
+
+        if (savepointFetchResult.getError() != null) {
+            currentSavepointInfo.resetTrigger();
+            return Optional.of(savepointFetchResult.getError());
         }
 
         LOG.info("Savepoint status updated with latest completed savepoint info");
         currentSavepointInfo.updateLastSavepoint(savepointFetchResult.getSavepoint());
+
         updateSavepointHistory(
                 currentSavepointInfo, savepointFetchResult.getSavepoint(), deployedConfig);
         return Optional.empty();
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java
index 9caa6e7..1015fb1 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java
@@ -98,6 +98,20 @@ public abstract class AbstractDeploymentObserver implements Observer<FlinkDeploy
             }
         }
 
+        if (!ReconciliationUtils.isJobRunning(flinkApp.getStatus())) {
+            if (flinkApp.getStatus().getJobStatus().getSavepointInfo().resetTrigger()) {
+                logger.error("Job is not running, cancelling savepoint operation");
+                EventUtils.createOrUpdateEvent(
+                        flinkService.getKubernetesClient(),
+                        flinkApp,
+                        EventUtils.Type.Warning,
+                        "SavepointError",
+                        "Savepoint failed for savepointTriggerNonce: "
+                                + flinkApp.getSpec().getJob().getSavepointTriggerNonce(),
+                        EventUtils.Component.Operator);
+            }
+        }
+
         clearErrorsIfDeploymentIsHealthy(flinkApp);
     }
 
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
index 124e6b5..d7c2756 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
@@ -134,6 +134,10 @@ public class FlinkService {
                         4, new ExecutorThreadFactory("Flink-RestClusterClient-IO"));
     }
 
+    public KubernetesClient getKubernetesClient() {
+        return kubernetesClient;
+    }
+
     public void submitApplicationCluster(
             JobSpec jobSpec, Configuration conf, boolean requireHaMetadata) throws Exception {
         LOG.info(
@@ -637,7 +641,7 @@ public class FlinkService {
     }
 
     public SavepointFetchResult fetchSavepointInfo(
-            String triggerId, String jobId, Configuration conf) throws Exception {
+            String triggerId, String jobId, Configuration conf) {
         LOG.info("Fetching savepoint result with triggerId: " + triggerId);
         try (RestClusterClient<String> clusterClient =
                 (RestClusterClient<String>) getClusterClient(conf)) {
@@ -654,24 +658,28 @@ public class FlinkService {
                             EmptyRequestBody.getInstance());
 
             if (response.get() == null || response.get().resource() == null) {
-                return SavepointFetchResult.notTriggered();
+                return SavepointFetchResult.pending();
             }
 
             if (response.get().resource().getLocation() == null) {
                 if (response.get().resource().getFailureCause() != null) {
-                    LOG.error("Savepoint error", response.get().resource().getFailureCause());
+                    LOG.error(
+                            "Failure occurred while fetching the savepoint result",
+                            response.get().resource().getFailureCause());
                     return SavepointFetchResult.error(
-                            response.get().resource().getFailureCause().getMessage());
+                            response.get().resource().getFailureCause().toString());
                 } else {
                     return SavepointFetchResult.pending();
                 }
             }
-
             Savepoint savepoint =
                     new Savepoint(
                             System.currentTimeMillis(), response.get().resource().getLocation());
             LOG.info("Savepoint result: " + savepoint);
             return SavepointFetchResult.completed(savepoint);
+        } catch (Exception e) {
+            LOG.error("Exception while fetching the savepoint result", e);
+            return SavepointFetchResult.error(e.getMessage());
         }
     }
 
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
index 2d28872..2540cf9 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
@@ -46,6 +46,7 @@ import org.apache.flink.runtime.rest.messages.DashboardConfiguration;
 
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.PodList;
+import io.fabric8.kubernetes.client.KubernetesClient;
 import io.javaoperatorsdk.operator.api.reconciler.Context;
 import io.javaoperatorsdk.operator.api.reconciler.RetryInfo;
 
@@ -69,9 +70,10 @@ public class TestingFlinkService extends FlinkService {
 
     public static final Map<String, String> CLUSTER_INFO =
             Map.of(
-                    DashboardConfiguration.FIELD_NAME_FLINK_VERSION, "15.0.0",
+                    DashboardConfiguration.FIELD_NAME_FLINK_VERSION,
+                    "15.0.0",
                     DashboardConfiguration.FIELD_NAME_FLINK_REVISION,
-                            "1234567 @ 1970-01-01T00:00:00+00:00");
+                    "1234567 @ 1970-01-01T00:00:00+00:00");
 
     private int savepointCounter = 0;
     private int triggerCounter = 0;
@@ -85,11 +87,16 @@ public class TestingFlinkService extends FlinkService {
     private PodList podList = new PodList();
     private Consumer<Configuration> listJobConsumer = conf -> {};
     private List<String> disposedSavepoints = new ArrayList<>();
+    private SavepointFetchResult savepointFetchResult;
 
     public TestingFlinkService() {
         super(null, new FlinkConfigManager(new Configuration()));
     }
 
+    public TestingFlinkService(KubernetesClient kubernetesClient) {
+        super(kubernetesClient, new FlinkConfigManager(new Configuration()));
+    }
+
     public Context getContext() {
         return new Context() {
             @Override
@@ -302,6 +309,17 @@ public class TestingFlinkService extends FlinkService {
     @Override
     public SavepointFetchResult fetchSavepointInfo(
             String triggerId, String jobId, Configuration conf) {
+
+        if (savepointFetchResult == null) {
+            savepointFetchResult = SavepointFetchResult.pending();
+            return savepointFetchResult;
+        }
+
+        if (savepointFetchResult.isPending()) {
+            savepointFetchResult = SavepointFetchResult.error("Failed");
+            return savepointFetchResult;
+        }
+
         return SavepointFetchResult.completed(Savepoint.of("savepoint_" + savepointCounter++));
     }
 
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java
index 78b1320..2211216 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java
@@ -29,9 +29,12 @@ import org.apache.flink.kubernetes.operator.crd.status.FlinkDeploymentStatus;
 import org.apache.flink.kubernetes.operator.crd.status.JobManagerDeploymentStatus;
 import org.apache.flink.kubernetes.operator.crd.status.JobStatus;
 import org.apache.flink.kubernetes.operator.exception.DeploymentFailedException;
+import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
 import org.apache.flink.kubernetes.operator.utils.SavepointUtils;
 import org.apache.flink.runtime.client.JobStatusMessage;
 
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
 import io.javaoperatorsdk.operator.api.reconciler.Context;
 import org.junit.jupiter.api.Test;
 
@@ -42,8 +45,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /** {@link ApplicationObserver} unit tests. */
+@EnableKubernetesMockClient(crud = true)
 public class ApplicationObserverTest {
-
+    private KubernetesClient kubernetesClient;
     private final Context readyContext = TestUtils.createContextWithReadyJobManagerDeployment();
     private final FlinkConfigManager configManager = new FlinkConfigManager(new Configuration());
 
@@ -156,7 +160,7 @@ public class ApplicationObserverTest {
 
     @Test
     public void observeSavepoint() throws Exception {
-        TestingFlinkService flinkService = new TestingFlinkService();
+        TestingFlinkService flinkService = new TestingFlinkService(kubernetesClient);
         ApplicationObserver observer =
                 new ApplicationObserver(
                         null, flinkService, configManager, new TestingStatusHelper<>());
@@ -165,19 +169,59 @@ public class ApplicationObserverTest {
                 configManager.getDeployConfig(deployment.getMetadata(), deployment.getSpec());
         flinkService.submitApplicationCluster(deployment.getSpec().getJob(), conf, false);
         bringToReadyStatus(deployment);
-        observer.observe(deployment, readyContext);
-        assertEquals(
-                JobManagerDeploymentStatus.READY,
-                deployment.getStatus().getJobManagerDeploymentStatus());
+        assertTrue(ReconciliationUtils.isJobRunning(deployment.getStatus()));
 
         flinkService.triggerSavepoint(
                 deployment.getStatus().getJobStatus().getJobId(),
                 deployment.getStatus().getJobStatus().getSavepointInfo(),
                 conf);
+        // pending savepoint
         assertEquals(
                 "trigger_0",
                 deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
+        observer.observe(deployment, readyContext);
+        assertTrue(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+
+        // savepoint error
+        assertEquals(
+                0,
+                kubernetesClient
+                        .v1()
+                        .events()
+                        .inNamespace(deployment.getMetadata().getNamespace())
+                        .list()
+                        .getItems()
+                        .size());
+        observer.observe(deployment, readyContext);
+        assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+        assertEquals(
+                1,
+                kubernetesClient
+                        .v1()
+                        .events()
+                        .inNamespace(deployment.getMetadata().getNamespace())
+                        .list()
+                        .getItems()
+                        .size());
+        assertEquals(
+                1,
+                kubernetesClient
+                        .v1()
+                        .events()
+                        .inNamespace(deployment.getMetadata().getNamespace())
+                        .list()
+                        .getItems()
+                        .get(0)
+                        .getCount());
 
+        // savepoint success
+        flinkService.triggerSavepoint(
+                deployment.getStatus().getJobStatus().getJobId(),
+                deployment.getStatus().getJobStatus().getSavepointInfo(),
+                conf);
+        assertEquals(
+                "trigger_1",
+                deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
         observer.observe(deployment, readyContext);
         assertEquals(
                 "savepoint_0",
@@ -187,15 +231,15 @@ public class ApplicationObserverTest {
                         .getSavepointInfo()
                         .getLastSavepoint()
                         .getLocation());
-
         assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
 
+        // second attempt success
         flinkService.triggerSavepoint(
                 deployment.getStatus().getJobStatus().getJobId(),
                 deployment.getStatus().getJobStatus().getSavepointInfo(),
                 conf);
         assertEquals(
-                "trigger_1",
+                "trigger_2",
                 deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
         assertTrue(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
 
@@ -210,6 +254,49 @@ public class ApplicationObserverTest {
                         .getLocation());
         assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
 
+        // application failure after checkpoint trigger
+        flinkService.triggerSavepoint(
+                deployment.getStatus().getJobStatus().getJobId(),
+                deployment.getStatus().getJobStatus().getSavepointInfo(),
+                conf);
+        assertEquals(
+                "trigger_3",
+                deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
+        assertTrue(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+        flinkService.setPortReady(false);
+        observer.observe(deployment, readyContext);
+        assertEquals(
+                "savepoint_1",
+                deployment
+                        .getStatus()
+                        .getJobStatus()
+                        .getSavepointInfo()
+                        .getLastSavepoint()
+                        .getLocation());
+        assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+
+        assertEquals(
+                1,
+                kubernetesClient
+                        .v1()
+                        .events()
+                        .inNamespace(deployment.getMetadata().getNamespace())
+                        .list()
+                        .getItems()
+                        .size());
+        assertEquals(
+                2,
+                kubernetesClient
+                        .v1()
+                        .events()
+                        .inNamespace(deployment.getMetadata().getNamespace())
+                        .list()
+                        .getItems()
+                        .get(0)
+                        .getCount());
+
+        flinkService.setPortReady(true);
+        observer.observe(deployment, readyContext);
         // Simulate Failed job
         Tuple2<String, JobStatusMessage> jobTuple = flinkService.listJobs().get(0);
         jobTuple.f0 = "last-SP";
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java
index c5ff4eb..667067f 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java
@@ -28,14 +28,17 @@ import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
 import org.apache.flink.kubernetes.operator.reconciler.sessionjob.FlinkSessionJobReconciler;
 import org.apache.flink.kubernetes.operator.utils.SavepointUtils;
 
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 import java.util.Map;
 
 /** Tests for {@link SessionJobObserver}. */
+@EnableKubernetesMockClient(crud = true)
 public class SessionJobObserverTest {
-
+    private KubernetesClient kubernetesClient;
     private final FlinkConfigManager configManager = new FlinkConfigManager(new Configuration());
 
     @Test
@@ -128,7 +131,7 @@ public class SessionJobObserverTest {
     @Test
     public void testObserveSavepoint() throws Exception {
         final var sessionJob = TestUtils.buildSessionJob();
-        final var flinkService = new TestingFlinkService();
+        final var flinkService = new TestingFlinkService(kubernetesClient);
         final var reconciler = new FlinkSessionJobReconciler(null, flinkService, configManager);
         final var observer =
                 new SessionJobObserver(flinkService, configManager, new TestingStatusHelper<>());
@@ -152,9 +155,18 @@ public class SessionJobObserverTest {
         flinkService.triggerSavepoint(jobID, savepointInfo, new Configuration());
         Assertions.assertTrue(
                 SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));
-
         Assertions.assertEquals("trigger_0", savepointInfo.getTriggerId());
-        observer.observe(sessionJob, readyContext);
+        observer.observe(sessionJob, readyContext); // pending
+        observer.observe(sessionJob, readyContext); // error
+
+        flinkService.triggerSavepoint(jobID, savepointInfo, new Configuration());
+        Assertions.assertTrue(
+                SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));
+        Assertions.assertEquals("trigger_1", savepointInfo.getTriggerId());
+        flinkService.triggerSavepoint(jobID, savepointInfo, new Configuration());
+        Assertions.assertTrue(
+                SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));
+        observer.observe(sessionJob, readyContext); // success
         Assertions.assertEquals("savepoint_0", savepointInfo.getLastSavepoint().getLocation());
         Assertions.assertFalse(
                 SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));