You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by wa...@apache.org on 2022/05/19 13:21:06 UTC
[flink-kubernetes-operator] branch release-1.0 updated: [FLINK-27675] Improve manual savepoint tracking
This is an automated email from the ASF dual-hosted git repository.
wangyang0918 pushed a commit to branch release-1.0
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
The following commit(s) were added to refs/heads/release-1.0 by this push:
new 851b905 [FLINK-27675] Improve manual savepoint tracking
851b905 is described below
commit 851b9059055c849bd4fd2d0c4293b6774a911354
Author: Matyas Orhidi <ma...@apple.com>
AuthorDate: Wed May 18 14:22:02 2022 +0200
[FLINK-27675] Improve manual savepoint tracking
---
.../operator/crd/status/SavepointInfo.java | 5 +-
.../operator/observer/SavepointFetchResult.java | 8 +-
.../operator/observer/SavepointObserver.java | 59 +++++++-----
.../deployment/AbstractDeploymentObserver.java | 14 +++
.../kubernetes/operator/service/FlinkService.java | 18 +++-
.../kubernetes/operator/TestingFlinkService.java | 22 ++++-
.../deployment/ApplicationObserverTest.java | 103 +++++++++++++++++++--
.../sessionjob/SessionJobObserverTest.java | 20 +++-
8 files changed, 199 insertions(+), 50 deletions(-)
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java
index b20691d..a2eb133 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/crd/status/SavepointInfo.java
@@ -23,6 +23,7 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
@@ -51,9 +52,11 @@ public class SavepointInfo {
this.triggerTimestamp = System.currentTimeMillis();
}
- public void resetTrigger() {
+ public boolean resetTrigger() {
+ boolean reseted = StringUtils.isNotEmpty(this.triggerId);
this.triggerId = "";
this.triggerTimestamp = 0L;
+ return reseted;
}
public void updateLastSavepoint(Savepoint savepoint) {
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java
index 74524c0..9f111fa 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointFetchResult.java
@@ -25,13 +25,9 @@ import lombok.Value;
@Value
public class SavepointFetchResult {
Savepoint savepoint;
- boolean isTriggered;
+ boolean pending;
String error;
- public static SavepointFetchResult notTriggered() {
- return new SavepointFetchResult(null, false, null);
- }
-
public static SavepointFetchResult error(String error) {
return new SavepointFetchResult(null, false, error);
}
@@ -41,6 +37,6 @@ public class SavepointFetchResult {
}
public static SavepointFetchResult completed(Savepoint savepoint) {
- return new SavepointFetchResult(savepoint, true, null);
+ return new SavepointFetchResult(savepoint, false, null);
}
}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java
index daa8142..91726d7 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/SavepointObserver.java
@@ -27,8 +27,8 @@ import org.apache.flink.kubernetes.operator.crd.status.CommonStatus;
import org.apache.flink.kubernetes.operator.crd.status.Savepoint;
import org.apache.flink.kubernetes.operator.crd.status.SavepointInfo;
import org.apache.flink.kubernetes.operator.exception.ReconciliationException;
-import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
import org.apache.flink.kubernetes.operator.service.FlinkService;
+import org.apache.flink.kubernetes.operator.utils.EventUtils;
import org.apache.flink.kubernetes.operator.utils.SavepointUtils;
import org.apache.flink.kubernetes.operator.utils.StatusHelper;
@@ -70,7 +70,18 @@ public class SavepointObserver<STATUS extends CommonStatus<?>> {
.orElse(null);
observeTriggeredSavepointProgress(savepointInfo, jobId, deployedConfig)
- .ifPresent(err -> ReconciliationUtils.updateForReconciliationError(resource, err));
+ .ifPresent(
+ err ->
+ EventUtils.createOrUpdateEvent(
+ flinkService.getKubernetesClient(),
+ resource,
+ EventUtils.Type.Warning,
+ "SavepointError",
+ "Savepoint failed for savepointTriggerNonce: "
+ + resource.getSpec()
+ .getJob()
+ .getSavepointTriggerNonce(),
+ EventUtils.Component.Operator));
// We only need to observe latest checkpoint/savepoint for terminal jobs
if (JobStatus.valueOf(jobStatus.getState()).isGloballyTerminalState()) {
@@ -106,36 +117,36 @@ public class SavepointObserver<STATUS extends CommonStatus<?>> {
LOG.debug("Savepoint not in progress");
return Optional.empty();
}
- LOG.info("Observing savepoint status");
- SavepointFetchResult savepointFetchResult;
- try {
- savepointFetchResult =
- flinkService.fetchSavepointInfo(
- currentSavepointInfo.getTriggerId(), jobID, deployedConfig);
- } catch (Exception e) {
- LOG.error("Exception while fetching savepoint info", e);
- return Optional.empty();
- }
-
- if (!savepointFetchResult.isTriggered()) {
- String error = savepointFetchResult.getError();
- if (error != null
- || SavepointUtils.gracePeriodEnded(
- configManager.getOperatorConfiguration(), currentSavepointInfo)) {
- String errorMsg = error != null ? error : "Savepoint status unknown";
- LOG.error(errorMsg);
+ LOG.info("Observing savepoint status.");
+ SavepointFetchResult savepointFetchResult =
+ flinkService.fetchSavepointInfo(
+ currentSavepointInfo.getTriggerId(), jobID, deployedConfig);
+
+ if (savepointFetchResult.isPending()) {
+ if (SavepointUtils.gracePeriodEnded(
+ configManager.getOperatorConfiguration(), currentSavepointInfo)) {
+ String errorMsg =
+ "Savepoint operation timed out after "
+ + configManager
+ .getOperatorConfiguration()
+ .getSavepointTriggerGracePeriod();
currentSavepointInfo.resetTrigger();
+ LOG.error(errorMsg);
return Optional.of(errorMsg);
+ } else {
+ LOG.info("Savepoint operation not finished yet, waiting within grace period...");
+ return Optional.empty();
}
- LOG.info("Savepoint operation not running, waiting within grace period...");
}
- if (savepointFetchResult.getSavepoint() == null) {
- LOG.info("Savepoint is still in progress...");
- return Optional.empty();
+
+ if (savepointFetchResult.getError() != null) {
+ currentSavepointInfo.resetTrigger();
+ return Optional.of(savepointFetchResult.getError());
}
LOG.info("Savepoint status updated with latest completed savepoint info");
currentSavepointInfo.updateLastSavepoint(savepointFetchResult.getSavepoint());
+
updateSavepointHistory(
currentSavepointInfo, savepointFetchResult.getSavepoint(), deployedConfig);
return Optional.empty();
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java
index 9caa6e7..1015fb1 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/observer/deployment/AbstractDeploymentObserver.java
@@ -98,6 +98,20 @@ public abstract class AbstractDeploymentObserver implements Observer<FlinkDeploy
}
}
+ if (!ReconciliationUtils.isJobRunning(flinkApp.getStatus())) {
+ if (flinkApp.getStatus().getJobStatus().getSavepointInfo().resetTrigger()) {
+ logger.error("Job is not running, cancelling savepoint operation");
+ EventUtils.createOrUpdateEvent(
+ flinkService.getKubernetesClient(),
+ flinkApp,
+ EventUtils.Type.Warning,
+ "SavepointError",
+ "Savepoint failed for savepointTriggerNonce: "
+ + flinkApp.getSpec().getJob().getSavepointTriggerNonce(),
+ EventUtils.Component.Operator);
+ }
+ }
+
clearErrorsIfDeploymentIsHealthy(flinkApp);
}
diff --git a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
index 124e6b5..d7c2756 100644
--- a/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
+++ b/flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/service/FlinkService.java
@@ -134,6 +134,10 @@ public class FlinkService {
4, new ExecutorThreadFactory("Flink-RestClusterClient-IO"));
}
+ public KubernetesClient getKubernetesClient() {
+ return kubernetesClient;
+ }
+
public void submitApplicationCluster(
JobSpec jobSpec, Configuration conf, boolean requireHaMetadata) throws Exception {
LOG.info(
@@ -637,7 +641,7 @@ public class FlinkService {
}
public SavepointFetchResult fetchSavepointInfo(
- String triggerId, String jobId, Configuration conf) throws Exception {
+ String triggerId, String jobId, Configuration conf) {
LOG.info("Fetching savepoint result with triggerId: " + triggerId);
try (RestClusterClient<String> clusterClient =
(RestClusterClient<String>) getClusterClient(conf)) {
@@ -654,24 +658,28 @@ public class FlinkService {
EmptyRequestBody.getInstance());
if (response.get() == null || response.get().resource() == null) {
- return SavepointFetchResult.notTriggered();
+ return SavepointFetchResult.pending();
}
if (response.get().resource().getLocation() == null) {
if (response.get().resource().getFailureCause() != null) {
- LOG.error("Savepoint error", response.get().resource().getFailureCause());
+ LOG.error(
+ "Failure occurred while fetching the savepoint result",
+ response.get().resource().getFailureCause());
return SavepointFetchResult.error(
- response.get().resource().getFailureCause().getMessage());
+ response.get().resource().getFailureCause().toString());
} else {
return SavepointFetchResult.pending();
}
}
-
Savepoint savepoint =
new Savepoint(
System.currentTimeMillis(), response.get().resource().getLocation());
LOG.info("Savepoint result: " + savepoint);
return SavepointFetchResult.completed(savepoint);
+ } catch (Exception e) {
+ LOG.error("Exception while fetching the savepoint result", e);
+ return SavepointFetchResult.error(e.getMessage());
}
}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
index 2d28872..2540cf9 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/TestingFlinkService.java
@@ -46,6 +46,7 @@ import org.apache.flink.runtime.rest.messages.DashboardConfiguration;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.PodList;
+import io.fabric8.kubernetes.client.KubernetesClient;
import io.javaoperatorsdk.operator.api.reconciler.Context;
import io.javaoperatorsdk.operator.api.reconciler.RetryInfo;
@@ -69,9 +70,10 @@ public class TestingFlinkService extends FlinkService {
public static final Map<String, String> CLUSTER_INFO =
Map.of(
- DashboardConfiguration.FIELD_NAME_FLINK_VERSION, "15.0.0",
+ DashboardConfiguration.FIELD_NAME_FLINK_VERSION,
+ "15.0.0",
DashboardConfiguration.FIELD_NAME_FLINK_REVISION,
- "1234567 @ 1970-01-01T00:00:00+00:00");
+ "1234567 @ 1970-01-01T00:00:00+00:00");
private int savepointCounter = 0;
private int triggerCounter = 0;
@@ -85,11 +87,16 @@ public class TestingFlinkService extends FlinkService {
private PodList podList = new PodList();
private Consumer<Configuration> listJobConsumer = conf -> {};
private List<String> disposedSavepoints = new ArrayList<>();
+ private SavepointFetchResult savepointFetchResult;
public TestingFlinkService() {
super(null, new FlinkConfigManager(new Configuration()));
}
+ public TestingFlinkService(KubernetesClient kubernetesClient) {
+ super(kubernetesClient, new FlinkConfigManager(new Configuration()));
+ }
+
public Context getContext() {
return new Context() {
@Override
@@ -302,6 +309,17 @@ public class TestingFlinkService extends FlinkService {
@Override
public SavepointFetchResult fetchSavepointInfo(
String triggerId, String jobId, Configuration conf) {
+
+ if (savepointFetchResult == null) {
+ savepointFetchResult = SavepointFetchResult.pending();
+ return savepointFetchResult;
+ }
+
+ if (savepointFetchResult.isPending()) {
+ savepointFetchResult = SavepointFetchResult.error("Failed");
+ return savepointFetchResult;
+ }
+
return SavepointFetchResult.completed(Savepoint.of("savepoint_" + savepointCounter++));
}
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java
index 78b1320..2211216 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/deployment/ApplicationObserverTest.java
@@ -29,9 +29,12 @@ import org.apache.flink.kubernetes.operator.crd.status.FlinkDeploymentStatus;
import org.apache.flink.kubernetes.operator.crd.status.JobManagerDeploymentStatus;
import org.apache.flink.kubernetes.operator.crd.status.JobStatus;
import org.apache.flink.kubernetes.operator.exception.DeploymentFailedException;
+import org.apache.flink.kubernetes.operator.reconciler.ReconciliationUtils;
import org.apache.flink.kubernetes.operator.utils.SavepointUtils;
import org.apache.flink.runtime.client.JobStatusMessage;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
import io.javaoperatorsdk.operator.api.reconciler.Context;
import org.junit.jupiter.api.Test;
@@ -42,8 +45,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** {@link ApplicationObserver} unit tests. */
+@EnableKubernetesMockClient(crud = true)
public class ApplicationObserverTest {
-
+ private KubernetesClient kubernetesClient;
private final Context readyContext = TestUtils.createContextWithReadyJobManagerDeployment();
private final FlinkConfigManager configManager = new FlinkConfigManager(new Configuration());
@@ -156,7 +160,7 @@ public class ApplicationObserverTest {
@Test
public void observeSavepoint() throws Exception {
- TestingFlinkService flinkService = new TestingFlinkService();
+ TestingFlinkService flinkService = new TestingFlinkService(kubernetesClient);
ApplicationObserver observer =
new ApplicationObserver(
null, flinkService, configManager, new TestingStatusHelper<>());
@@ -165,19 +169,59 @@ public class ApplicationObserverTest {
configManager.getDeployConfig(deployment.getMetadata(), deployment.getSpec());
flinkService.submitApplicationCluster(deployment.getSpec().getJob(), conf, false);
bringToReadyStatus(deployment);
- observer.observe(deployment, readyContext);
- assertEquals(
- JobManagerDeploymentStatus.READY,
- deployment.getStatus().getJobManagerDeploymentStatus());
+ assertTrue(ReconciliationUtils.isJobRunning(deployment.getStatus()));
flinkService.triggerSavepoint(
deployment.getStatus().getJobStatus().getJobId(),
deployment.getStatus().getJobStatus().getSavepointInfo(),
conf);
+ // pending savepoint
assertEquals(
"trigger_0",
deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
+ observer.observe(deployment, readyContext);
+ assertTrue(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+
+ // savepoint error
+ assertEquals(
+ 0,
+ kubernetesClient
+ .v1()
+ .events()
+ .inNamespace(deployment.getMetadata().getNamespace())
+ .list()
+ .getItems()
+ .size());
+ observer.observe(deployment, readyContext);
+ assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+ assertEquals(
+ 1,
+ kubernetesClient
+ .v1()
+ .events()
+ .inNamespace(deployment.getMetadata().getNamespace())
+ .list()
+ .getItems()
+ .size());
+ assertEquals(
+ 1,
+ kubernetesClient
+ .v1()
+ .events()
+ .inNamespace(deployment.getMetadata().getNamespace())
+ .list()
+ .getItems()
+ .get(0)
+ .getCount());
+ // savepoint success
+ flinkService.triggerSavepoint(
+ deployment.getStatus().getJobStatus().getJobId(),
+ deployment.getStatus().getJobStatus().getSavepointInfo(),
+ conf);
+ assertEquals(
+ "trigger_1",
+ deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
observer.observe(deployment, readyContext);
assertEquals(
"savepoint_0",
@@ -187,15 +231,15 @@ public class ApplicationObserverTest {
.getSavepointInfo()
.getLastSavepoint()
.getLocation());
-
assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+ // second attempt success
flinkService.triggerSavepoint(
deployment.getStatus().getJobStatus().getJobId(),
deployment.getStatus().getJobStatus().getSavepointInfo(),
conf);
assertEquals(
- "trigger_1",
+ "trigger_2",
deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
assertTrue(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
@@ -210,6 +254,49 @@ public class ApplicationObserverTest {
.getLocation());
assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+ // application failure after checkpoint trigger
+ flinkService.triggerSavepoint(
+ deployment.getStatus().getJobStatus().getJobId(),
+ deployment.getStatus().getJobStatus().getSavepointInfo(),
+ conf);
+ assertEquals(
+ "trigger_3",
+ deployment.getStatus().getJobStatus().getSavepointInfo().getTriggerId());
+ assertTrue(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+ flinkService.setPortReady(false);
+ observer.observe(deployment, readyContext);
+ assertEquals(
+ "savepoint_1",
+ deployment
+ .getStatus()
+ .getJobStatus()
+ .getSavepointInfo()
+ .getLastSavepoint()
+ .getLocation());
+ assertFalse(SavepointUtils.savepointInProgress(deployment.getStatus().getJobStatus()));
+
+ assertEquals(
+ 1,
+ kubernetesClient
+ .v1()
+ .events()
+ .inNamespace(deployment.getMetadata().getNamespace())
+ .list()
+ .getItems()
+ .size());
+ assertEquals(
+ 2,
+ kubernetesClient
+ .v1()
+ .events()
+ .inNamespace(deployment.getMetadata().getNamespace())
+ .list()
+ .getItems()
+ .get(0)
+ .getCount());
+
+ flinkService.setPortReady(true);
+ observer.observe(deployment, readyContext);
// Simulate Failed job
Tuple2<String, JobStatusMessage> jobTuple = flinkService.listJobs().get(0);
jobTuple.f0 = "last-SP";
diff --git a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java
index c5ff4eb..667067f 100644
--- a/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java
+++ b/flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/observer/sessionjob/SessionJobObserverTest.java
@@ -28,14 +28,17 @@ import org.apache.flink.kubernetes.operator.config.FlinkConfigManager;
import org.apache.flink.kubernetes.operator.reconciler.sessionjob.FlinkSessionJobReconciler;
import org.apache.flink.kubernetes.operator.utils.SavepointUtils;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.Map;
/** Tests for {@link SessionJobObserver}. */
+@EnableKubernetesMockClient(crud = true)
public class SessionJobObserverTest {
-
+ private KubernetesClient kubernetesClient;
private final FlinkConfigManager configManager = new FlinkConfigManager(new Configuration());
@Test
@@ -128,7 +131,7 @@ public class SessionJobObserverTest {
@Test
public void testObserveSavepoint() throws Exception {
final var sessionJob = TestUtils.buildSessionJob();
- final var flinkService = new TestingFlinkService();
+ final var flinkService = new TestingFlinkService(kubernetesClient);
final var reconciler = new FlinkSessionJobReconciler(null, flinkService, configManager);
final var observer =
new SessionJobObserver(flinkService, configManager, new TestingStatusHelper<>());
@@ -152,9 +155,18 @@ public class SessionJobObserverTest {
flinkService.triggerSavepoint(jobID, savepointInfo, new Configuration());
Assertions.assertTrue(
SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));
-
Assertions.assertEquals("trigger_0", savepointInfo.getTriggerId());
- observer.observe(sessionJob, readyContext);
+ observer.observe(sessionJob, readyContext); // pending
+ observer.observe(sessionJob, readyContext); // error
+
+ flinkService.triggerSavepoint(jobID, savepointInfo, new Configuration());
+ Assertions.assertTrue(
+ SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));
+ Assertions.assertEquals("trigger_1", savepointInfo.getTriggerId());
+ flinkService.triggerSavepoint(jobID, savepointInfo, new Configuration());
+ Assertions.assertTrue(
+ SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));
+ observer.observe(sessionJob, readyContext); // success
Assertions.assertEquals("savepoint_0", savepointInfo.getLastSavepoint().getLocation());
Assertions.assertFalse(
SavepointUtils.savepointInProgress(sessionJob.getStatus().getJobStatus()));