You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2021/11/05 08:45:56 UTC

[flink] branch master updated (0a5ca14 -> 3a56132)

This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from 0a5ca14  [FLINK-24770] Remove error from JobStatusListener
     new 95eca7e  [hotfix] Fix formatting issues in generated files
     new e8b11aa  [FLINK-24208][rest] Support user-supplied Savepoint triggerId
     new 9d707b3  [FLINK-24208][rest] Consistent nullable getters
     new 3a56132  [FLINK-24208][rest] Add StopWithSavepointTriggerRequestBodyTest

The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 docs/content/docs/ops/rest_api.md                  |  7 +++
 .../shortcodes/generated/rest_v1_dispatcher.html   | 12 +++--
 .../client/program/rest/RestClusterClient.java     |  5 ++-
 .../RestClusterClientSavepointTriggerTest.java     |  6 +--
 .../src/test/resources/rest_api_v1.snapshot        |  8 +++-
 .../handler/job/savepoints/SavepointHandlers.java  | 36 +++++++++------
 .../savepoints/SavepointTriggerRequestBody.java    | 25 +++++++++--
 .../stop/StopWithSavepointRequestBody.java         | 25 +++++++++--
 .../job/savepoints/SavepointHandlersTest.java      | 52 ++++++++++++++++++++--
 .../savepoints/StopWithSavepointHandlersTest.java  | 52 ++++++++++++++++++++--
 .../messages/SavepointHandlerRequestBodyTest.java  |  5 +--
 .../SavepointTriggerRequestBodyTest.java           |  8 +++-
 ...> StopWithSavepointTriggerRequestBodyTest.java} | 31 ++++++++-----
 13 files changed, 217 insertions(+), 55 deletions(-)
 copy flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/{SavepointTriggerRequestBodyTest.java => StopWithSavepointTriggerRequestBodyTest.java} (55%)

[flink] 03/04: [FLINK-24208][rest] Consistent nullable getters

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 9d707b309dde763c07811d4b9e4a9923c7700741
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Nov 4 09:47:35 2021 +0100

    [FLINK-24208][rest] Consistent nullable getters
---
 .../rest/RestClusterClientSavepointTriggerTest.java  |  6 +++---
 .../handler/job/savepoints/SavepointHandlers.java    | 20 ++++++++------------
 .../job/savepoints/SavepointTriggerRequestBody.java  |  6 +++---
 .../stop/StopWithSavepointRequestBody.java           |  6 +++---
 .../job/savepoints/SavepointHandlersTest.java        |  2 +-
 .../savepoints/StopWithSavepointHandlersTest.java    |  2 +-
 .../messages/SavepointHandlerRequestBodyTest.java    |  5 ++---
 7 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientSavepointTriggerTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientSavepointTriggerTest.java
index 061abea..5310e85 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientSavepointTriggerTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientSavepointTriggerTest.java
@@ -70,11 +70,11 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
@@ -129,7 +129,7 @@ public class RestClusterClientSavepointTriggerTest extends TestLogger {
         try (final RestServerEndpoint restServerEndpoint =
                 createRestServerEndpoint(
                         request -> {
-                            assertNull(request.getTargetDirectory());
+                            assertThat(request.getTargetDirectory().isPresent(), is(false));
                             assertFalse(request.isCancelJob());
                             return triggerId;
                         },
@@ -158,7 +158,7 @@ public class RestClusterClientSavepointTriggerTest extends TestLogger {
                         triggerRequestBody -> {
                             assertEquals(
                                     expectedSubmittedSavepointDir,
-                                    triggerRequestBody.getTargetDirectory());
+                                    triggerRequestBody.getTargetDirectory().get());
                             assertFalse(triggerRequestBody.isCancelJob());
                             return triggerId;
                         },
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java
index 6ce3bb3..97cb1d0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java
@@ -179,9 +179,10 @@ public class SavepointHandlers {
                 AsynchronousJobOperationKey operationKey,
                 final RestfulGateway gateway)
                 throws RestHandlerException {
-            final String requestedTargetDirectory = request.getRequestBody().getTargetDirectory();
+            final Optional<String> requestedTargetDirectory =
+                    request.getRequestBody().getTargetDirectory();
 
-            if (requestedTargetDirectory == null && defaultSavepointDir == null) {
+            if (!requestedTargetDirectory.isPresent() && defaultSavepointDir == null) {
                 throw new RestHandlerException(
                         String.format(
                                 "Config key [%s] is not set. Property [%s] must be provided.",
@@ -194,10 +195,7 @@ public class SavepointHandlers {
                     request.getRequestBody().shouldDrain()
                             ? TriggerSavepointMode.TERMINATE_WITH_SAVEPOINT
                             : TriggerSavepointMode.SUSPEND_WITH_SAVEPOINT;
-            final String targetDirectory =
-                    requestedTargetDirectory != null
-                            ? requestedTargetDirectory
-                            : defaultSavepointDir;
+            final String targetDirectory = requestedTargetDirectory.orElse(defaultSavepointDir);
             return gateway.stopWithSavepoint(
                     operationKey, targetDirectory, savepointMode, RpcUtils.INF_TIMEOUT);
         }
@@ -224,9 +222,10 @@ public class SavepointHandlers {
                 AsynchronousJobOperationKey operationKey,
                 RestfulGateway gateway)
                 throws RestHandlerException {
-            final String requestedTargetDirectory = request.getRequestBody().getTargetDirectory();
+            final Optional<String> requestedTargetDirectory =
+                    request.getRequestBody().getTargetDirectory();
 
-            if (requestedTargetDirectory == null && defaultSavepointDir == null) {
+            if (!requestedTargetDirectory.isPresent() && defaultSavepointDir == null) {
                 throw new RestHandlerException(
                         String.format(
                                 "Config key [%s] is not set. Property [%s] must be provided.",
@@ -239,10 +238,7 @@ public class SavepointHandlers {
                     request.getRequestBody().isCancelJob()
                             ? TriggerSavepointMode.CANCEL_WITH_SAVEPOINT
                             : TriggerSavepointMode.SAVEPOINT;
-            final String targetDirectory =
-                    requestedTargetDirectory != null
-                            ? requestedTargetDirectory
-                            : defaultSavepointDir;
+            final String targetDirectory = requestedTargetDirectory.orElse(defaultSavepointDir);
             return gateway.triggerSavepoint(
                     operationKey, targetDirectory, savepointMode, RpcUtils.INF_TIMEOUT);
         }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java
index 6ae61be..ed5ee1e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java
@@ -59,9 +59,9 @@ public class SavepointTriggerRequestBody implements RequestBody {
         this.triggerId = triggerId;
     }
 
-    @Nullable
-    public String getTargetDirectory() {
-        return targetDirectory;
+    @JsonIgnore
+    public Optional<String> getTargetDirectory() {
+        return Optional.ofNullable(targetDirectory);
     }
 
     @JsonIgnore
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java
index a75bab7..cf80526 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java
@@ -59,9 +59,9 @@ public class StopWithSavepointRequestBody implements RequestBody {
         this.triggerId = triggerId;
     }
 
-    @Nullable
-    public String getTargetDirectory() {
-        return targetDirectory;
+    @JsonIgnore
+    public Optional<String> getTargetDirectory() {
+        return Optional.ofNullable(targetDirectory);
     }
 
     public boolean shouldDrain() {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java
index 468495f..95b7c36 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java
@@ -262,7 +262,7 @@ public class SavepointHandlersTest extends TestLogger {
     }
 
     private static HandlerRequest<SavepointTriggerRequestBody> triggerSavepointRequest(
-            final String targetDirectory, @Nullable TriggerId triggerId)
+            @Nullable final String targetDirectory, @Nullable TriggerId triggerId)
             throws HandlerRequestException {
         return HandlerRequest.resolveParametersAndCreate(
                 new SavepointTriggerRequestBody(targetDirectory, false, triggerId),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java
index feb7ed8..a2db348 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java
@@ -267,7 +267,7 @@ public class StopWithSavepointHandlersTest extends TestLogger {
     }
 
     private static HandlerRequest<StopWithSavepointRequestBody> triggerSavepointRequest(
-            final String targetDirectory, @Nullable TriggerId triggerId)
+            @Nullable final String targetDirectory, @Nullable TriggerId triggerId)
             throws HandlerRequestException {
         return HandlerRequest.resolveParametersAndCreate(
                 new StopWithSavepointRequestBody(targetDirectory, false, triggerId),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/SavepointHandlerRequestBodyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/SavepointHandlerRequestBodyTest.java
index 75daf85..40b6eef 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/SavepointHandlerRequestBodyTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/SavepointHandlerRequestBodyTest.java
@@ -27,7 +27,6 @@ import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMap
 import org.junit.Test;
 
 import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.nullValue;
 import static org.junit.Assert.assertThat;
 
 /** Tests for the savepoint request bodies. */
@@ -40,7 +39,7 @@ public class SavepointHandlerRequestBodyTest {
 
         assertThat(defaultParseResult.isCancelJob(), is(false));
 
-        assertThat(defaultParseResult.getTargetDirectory(), nullValue());
+        assertThat(defaultParseResult.getTargetDirectory().isPresent(), is(false));
     }
 
     @Test
@@ -51,7 +50,7 @@ public class SavepointHandlerRequestBodyTest {
 
         assertThat(defaultParseResult.shouldDrain(), is(false));
 
-        assertThat(defaultParseResult.getTargetDirectory(), nullValue());
+        assertThat(defaultParseResult.getTargetDirectory().isPresent(), is(false));
     }
 
     private static <T> T getDefaultParseResult(Class<T> clazz) throws JsonProcessingException {

[flink] 02/04: [FLINK-24208][rest] Support user-supplied Savepoint triggerId

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit e8b11aaaef770a27d638ed2a8a101c98d7b47dc5
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Fri Sep 10 08:31:21 2021 +0200

    [FLINK-24208][rest] Support user-supplied Savepoint triggerId
---
 docs/content/docs/ops/rest_api.md                  |  7 +++
 .../shortcodes/generated/rest_v1_dispatcher.html   |  6 +++
 .../client/program/rest/RestClusterClient.java     |  5 ++-
 .../src/test/resources/rest_api_v1.snapshot        |  6 +++
 .../handler/job/savepoints/SavepointHandlers.java  | 16 ++++++-
 .../savepoints/SavepointTriggerRequestBody.java    | 19 +++++++-
 .../stop/StopWithSavepointRequestBody.java         | 19 +++++++-
 .../job/savepoints/SavepointHandlersTest.java      | 52 ++++++++++++++++++++--
 .../savepoints/StopWithSavepointHandlersTest.java  | 52 ++++++++++++++++++++--
 .../SavepointTriggerRequestBodyTest.java           |  8 +++-
 10 files changed, 175 insertions(+), 15 deletions(-)

diff --git a/docs/content/docs/ops/rest_api.md b/docs/content/docs/ops/rest_api.md
index 205ce37..1f45fd8 100644
--- a/docs/content/docs/ops/rest_api.md
+++ b/docs/content/docs/ops/rest_api.md
@@ -66,6 +66,13 @@ Querying unsupported/non-existing versions will return a 404 error.
 
 There exist several async operations among these APIs, e.g. `trigger savepoint`, `rescale a job`. They would return a `triggerid` to identify the operation you just POST and then you need to use that `triggerid` to query for the status of the operation.
 
+For (stop-with-)savepoint operations you can control this `triggerId` by setting it in the body of the request that triggers the operation.
+This allow you to safely* retry such operations without triggering multiple savepoints.
+
+{{< hint info >}}
+The retry is only safe until the [async operation store duration]({{< ref "docs/deployment/config#rest-async-store-duration" >}}) has elapsed.
+{{</ hint >}}
+
 {{< tabs "f00ed142-b05f-44f0-bafc-799080c1d40d" >}}
 {{< tab "v1" >}}
 #### JobManager
diff --git a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
index bcb1b44..4e4b8ac 100644
--- a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
+++ b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
@@ -3260,6 +3260,9 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
     },
     "target-directory" : {
       "type" : "string"
+    },
+    "triggerId" : {
+      "type" : "any"
     }
   }
 }            </code>
@@ -3432,6 +3435,9 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
     },
     "targetDirectory" : {
       "type" : "string"
+    },
+    "triggerId" : {
+      "type" : "any"
     }
   }
 }            </code>
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
index 09c6de5..2a5d578 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
@@ -463,7 +463,8 @@ public class RestClusterClient<T> implements ClusterClient<T> {
                 sendRequest(
                         stopWithSavepointTriggerHeaders,
                         stopWithSavepointTriggerMessageParameters,
-                        new StopWithSavepointRequestBody(savepointDirectory, advanceToEndOfTime));
+                        new StopWithSavepointRequestBody(
+                                savepointDirectory, advanceToEndOfTime, null));
 
         return responseFuture
                 .thenCompose(
@@ -536,7 +537,7 @@ public class RestClusterClient<T> implements ClusterClient<T> {
                 sendRequest(
                         savepointTriggerHeaders,
                         savepointTriggerMessageParameters,
-                        new SavepointTriggerRequestBody(savepointDirectory, cancelJob));
+                        new SavepointTriggerRequestBody(savepointDirectory, cancelJob, null));
 
         return responseFuture
                 .thenCompose(
diff --git a/flink-runtime-web/src/test/resources/rest_api_v1.snapshot b/flink-runtime-web/src/test/resources/rest_api_v1.snapshot
index 083c081..a708e63 100644
--- a/flink-runtime-web/src/test/resources/rest_api_v1.snapshot
+++ b/flink-runtime-web/src/test/resources/rest_api_v1.snapshot
@@ -1874,6 +1874,9 @@
         },
         "cancel-job" : {
           "type" : "boolean"
+        },
+        "triggerId" : {
+          "type" : "any"
         }
       }
     },
@@ -1946,6 +1949,9 @@
         },
         "drain" : {
           "type" : "boolean"
+        },
+        "triggerId" : {
+          "type" : "any"
         }
       }
     },
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java
index 7aefe4a..6ce3bb3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlers.java
@@ -54,6 +54,7 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 
 /**
@@ -130,9 +131,12 @@ public class SavepointHandlers {
 
         protected AsynchronousJobOperationKey createOperationKey(final HandlerRequest<B> request) {
             final JobID jobId = request.getPathParameter(JobIDPathParameter.class);
-            return AsynchronousJobOperationKey.of(new TriggerId(), jobId);
+            return AsynchronousJobOperationKey.of(
+                    extractTriggerId(request.getRequestBody()).orElseGet(TriggerId::new), jobId);
         }
 
+        protected abstract Optional<TriggerId> extractTriggerId(B request);
+
         public CompletableFuture<TriggerResponse> handleRequest(
                 @Nonnull HandlerRequest<B> request, @Nonnull RestfulGateway gateway)
                 throws RestHandlerException {
@@ -165,6 +169,11 @@ public class SavepointHandlers {
         }
 
         @Override
+        protected Optional<TriggerId> extractTriggerId(StopWithSavepointRequestBody request) {
+            return request.getTriggerId();
+        }
+
+        @Override
         protected CompletableFuture<Acknowledge> triggerOperation(
                 final HandlerRequest<StopWithSavepointRequestBody> request,
                 AsynchronousJobOperationKey operationKey,
@@ -205,6 +214,11 @@ public class SavepointHandlers {
         }
 
         @Override
+        protected Optional<TriggerId> extractTriggerId(SavepointTriggerRequestBody request) {
+            return request.getTriggerId();
+        }
+
+        @Override
         protected CompletableFuture<Acknowledge> triggerOperation(
                 HandlerRequest<SavepointTriggerRequestBody> request,
                 AsynchronousJobOperationKey operationKey,
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java
index a465ed0..6ae61be 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBody.java
@@ -19,12 +19,16 @@
 package org.apache.flink.runtime.rest.messages.job.savepoints;
 
 import org.apache.flink.runtime.rest.messages.RequestBody;
+import org.apache.flink.runtime.rest.messages.TriggerId;
 
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
 
 import javax.annotation.Nullable;
 
+import java.util.Optional;
+
 /** {@link RequestBody} to trigger savepoints. */
 public class SavepointTriggerRequestBody implements RequestBody {
 
@@ -32,6 +36,8 @@ public class SavepointTriggerRequestBody implements RequestBody {
 
     private static final String FIELD_NAME_CANCEL_JOB = "cancel-job";
 
+    private static final String FIELD_NAME_TRIGGER_ID = "triggerId";
+
     @JsonProperty(FIELD_NAME_TARGET_DIRECTORY)
     @Nullable
     private final String targetDirectory;
@@ -39,12 +45,18 @@ public class SavepointTriggerRequestBody implements RequestBody {
     @JsonProperty(FIELD_NAME_CANCEL_JOB)
     private final boolean cancelJob;
 
+    @JsonProperty(FIELD_NAME_TRIGGER_ID)
+    @Nullable
+    private final TriggerId triggerId;
+
     @JsonCreator
     public SavepointTriggerRequestBody(
             @Nullable @JsonProperty(FIELD_NAME_TARGET_DIRECTORY) final String targetDirectory,
-            @Nullable @JsonProperty(FIELD_NAME_CANCEL_JOB) final Boolean cancelJob) {
+            @Nullable @JsonProperty(FIELD_NAME_CANCEL_JOB) final Boolean cancelJob,
+            @Nullable @JsonProperty(FIELD_NAME_TRIGGER_ID) TriggerId triggerId) {
         this.targetDirectory = targetDirectory;
         this.cancelJob = cancelJob != null ? cancelJob : false;
+        this.triggerId = triggerId;
     }
 
     @Nullable
@@ -52,6 +64,11 @@ public class SavepointTriggerRequestBody implements RequestBody {
         return targetDirectory;
     }
 
+    @JsonIgnore
+    public Optional<TriggerId> getTriggerId() {
+        return Optional.ofNullable(triggerId);
+    }
+
     public boolean isCancelJob() {
         return cancelJob;
     }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java
index 170e8bc..a75bab7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/job/savepoints/stop/StopWithSavepointRequestBody.java
@@ -19,12 +19,16 @@
 package org.apache.flink.runtime.rest.messages.job.savepoints.stop;
 
 import org.apache.flink.runtime.rest.messages.RequestBody;
+import org.apache.flink.runtime.rest.messages.TriggerId;
 
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonIgnore;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
 
 import javax.annotation.Nullable;
 
+import java.util.Optional;
+
 /** {@link RequestBody} for stopping a job with a savepoint. */
 public class StopWithSavepointRequestBody implements RequestBody {
 
@@ -32,6 +36,8 @@ public class StopWithSavepointRequestBody implements RequestBody {
 
     private static final String FIELD_NAME_DRAIN = "drain";
 
+    private static final String FIELD_NAME_TRIGGER_ID = "triggerId";
+
     @JsonProperty(FIELD_NAME_TARGET_DIRECTORY)
     @Nullable
     private final String targetDirectory;
@@ -39,12 +45,18 @@ public class StopWithSavepointRequestBody implements RequestBody {
     @JsonProperty(FIELD_NAME_DRAIN)
     private final boolean drain;
 
+    @JsonProperty(FIELD_NAME_TRIGGER_ID)
+    @Nullable
+    private final TriggerId triggerId;
+
     @JsonCreator
     public StopWithSavepointRequestBody(
             @Nullable @JsonProperty(FIELD_NAME_TARGET_DIRECTORY) final String targetDirectory,
-            @Nullable @JsonProperty(FIELD_NAME_DRAIN) final Boolean drain) {
+            @Nullable @JsonProperty(FIELD_NAME_DRAIN) final Boolean drain,
+            @Nullable @JsonProperty(FIELD_NAME_TRIGGER_ID) TriggerId triggerId) {
         this.targetDirectory = targetDirectory;
         this.drain = drain != null ? drain : false;
+        this.triggerId = triggerId;
     }
 
     @Nullable
@@ -55,4 +67,9 @@ public class StopWithSavepointRequestBody implements RequestBody {
     public boolean shouldDrain() {
         return drain;
     }
+
+    @JsonIgnore
+    public Optional<TriggerId> getTriggerId() {
+        return Optional.ofNullable(triggerId);
+    }
 }
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java
index 74bfe6d..468495f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/SavepointHandlersTest.java
@@ -46,6 +46,8 @@ import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseSt
 import org.junit.Before;
 import org.junit.Test;
 
+import javax.annotation.Nullable;
+
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -57,6 +59,7 @@ import static org.apache.flink.runtime.rest.handler.job.savepoints.SavepointTest
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
 
@@ -208,20 +211,61 @@ public class SavepointHandlersTest extends TestLogger {
         assertThat(savepointError, instanceOf(RuntimeException.class));
     }
 
+    @Test
+    public void testProvidedTriggerId() throws Exception {
+        final OperationResult<String> successfulResult =
+                OperationResult.success(COMPLETED_SAVEPOINT_EXTERNAL_POINTER);
+        AtomicReference<AsynchronousJobOperationKey> keyReference = new AtomicReference<>();
+        final TestingRestfulGateway testingRestfulGateway =
+                new TestingRestfulGateway.Builder()
+                        .setTriggerSavepointFunction(setReferenceToOperationKey(keyReference))
+                        .setGetSavepointStatusFunction(
+                                getResultIfKeyMatches(successfulResult, keyReference))
+                        .build();
+
+        final TriggerId providedTriggerId = new TriggerId();
+
+        final TriggerId returnedTriggerId =
+                savepointTriggerHandler
+                        .handleRequest(
+                                triggerSavepointRequest(
+                                        DEFAULT_REQUESTED_SAVEPOINT_TARGET_DIRECTORY,
+                                        providedTriggerId),
+                                testingRestfulGateway)
+                        .get()
+                        .getTriggerId();
+
+        assertEquals(providedTriggerId, returnedTriggerId);
+
+        AsynchronousOperationResult<SavepointInfo> savepointResponseBody;
+        savepointResponseBody =
+                savepointStatusHandler
+                        .handleRequest(
+                                savepointStatusRequest(providedTriggerId), testingRestfulGateway)
+                        .get();
+
+        assertThat(savepointResponseBody.queueStatus().getId(), equalTo(QueueStatus.Id.COMPLETED));
+        assertThat(savepointResponseBody.resource(), notNullValue());
+        assertThat(
+                savepointResponseBody.resource().getLocation(),
+                equalTo(COMPLETED_SAVEPOINT_EXTERNAL_POINTER));
+    }
+
     private static HandlerRequest<SavepointTriggerRequestBody> triggerSavepointRequest()
             throws HandlerRequestException {
-        return triggerSavepointRequest(DEFAULT_REQUESTED_SAVEPOINT_TARGET_DIRECTORY);
+        return triggerSavepointRequest(DEFAULT_REQUESTED_SAVEPOINT_TARGET_DIRECTORY, null);
     }
 
     private static HandlerRequest<SavepointTriggerRequestBody>
             triggerSavepointRequestWithDefaultDirectory() throws HandlerRequestException {
-        return triggerSavepointRequest(null);
+        return triggerSavepointRequest(null, null);
     }
 
     private static HandlerRequest<SavepointTriggerRequestBody> triggerSavepointRequest(
-            final String targetDirectory) throws HandlerRequestException {
+            final String targetDirectory, @Nullable TriggerId triggerId)
+            throws HandlerRequestException {
         return HandlerRequest.resolveParametersAndCreate(
-                new SavepointTriggerRequestBody(targetDirectory, false),
+                new SavepointTriggerRequestBody(targetDirectory, false, triggerId),
                 new SavepointTriggerMessageParameters(),
                 Collections.singletonMap(JobIDPathParameter.KEY, JOB_ID.toString()),
                 Collections.emptyMap(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java
index 23b5deb..feb7ed8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/savepoints/StopWithSavepointHandlersTest.java
@@ -46,6 +46,8 @@ import org.apache.flink.shaded.netty4.io.netty.handler.codec.http.HttpResponseSt
 import org.junit.Before;
 import org.junit.Test;
 
+import javax.annotation.Nullable;
+
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -57,6 +59,7 @@ import static org.apache.flink.runtime.rest.handler.job.savepoints.SavepointTest
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.fail;
 
@@ -213,20 +216,61 @@ public class StopWithSavepointHandlersTest extends TestLogger {
         assertThat(savepointError, instanceOf(RuntimeException.class));
     }
 
+    @Test
+    public void testProvidedTriggerId() throws Exception {
+        final OperationResult<String> successfulResult =
+                OperationResult.success(COMPLETED_SAVEPOINT_EXTERNAL_POINTER);
+        AtomicReference<AsynchronousJobOperationKey> keyReference = new AtomicReference<>();
+        final TestingRestfulGateway testingRestfulGateway =
+                new TestingRestfulGateway.Builder()
+                        .setStopWithSavepointFunction(setReferenceToOperationKey(keyReference))
+                        .setGetSavepointStatusFunction(
+                                getResultIfKeyMatches(successfulResult, keyReference))
+                        .build();
+
+        final TriggerId providedTriggerId = new TriggerId();
+
+        final TriggerId returnedTriggerId =
+                savepointTriggerHandler
+                        .handleRequest(
+                                triggerSavepointRequest(
+                                        DEFAULT_REQUESTED_SAVEPOINT_TARGET_DIRECTORY,
+                                        providedTriggerId),
+                                testingRestfulGateway)
+                        .get()
+                        .getTriggerId();
+
+        assertEquals(providedTriggerId, returnedTriggerId);
+
+        AsynchronousOperationResult<SavepointInfo> savepointResponseBody;
+        savepointResponseBody =
+                savepointStatusHandler
+                        .handleRequest(
+                                savepointStatusRequest(providedTriggerId), testingRestfulGateway)
+                        .get();
+
+        assertThat(savepointResponseBody.queueStatus().getId(), equalTo(QueueStatus.Id.COMPLETED));
+        assertThat(savepointResponseBody.resource(), notNullValue());
+        assertThat(
+                savepointResponseBody.resource().getLocation(),
+                equalTo(COMPLETED_SAVEPOINT_EXTERNAL_POINTER));
+    }
+
     private static HandlerRequest<StopWithSavepointRequestBody> triggerSavepointRequest()
             throws HandlerRequestException {
-        return triggerSavepointRequest(DEFAULT_REQUESTED_SAVEPOINT_TARGET_DIRECTORY);
+        return triggerSavepointRequest(DEFAULT_REQUESTED_SAVEPOINT_TARGET_DIRECTORY, null);
     }
 
     private static HandlerRequest<StopWithSavepointRequestBody>
             triggerSavepointRequestWithDefaultDirectory() throws HandlerRequestException {
-        return triggerSavepointRequest(null);
+        return triggerSavepointRequest(null, null);
     }
 
     private static HandlerRequest<StopWithSavepointRequestBody> triggerSavepointRequest(
-            final String targetDirectory) throws HandlerRequestException {
+            final String targetDirectory, @Nullable TriggerId triggerId)
+            throws HandlerRequestException {
         return HandlerRequest.resolveParametersAndCreate(
-                new StopWithSavepointRequestBody(targetDirectory, false),
+                new StopWithSavepointRequestBody(targetDirectory, false, triggerId),
                 new SavepointTriggerMessageParameters(),
                 Collections.singletonMap(JobIDPathParameter.KEY, JOB_ID.toString()),
                 Collections.emptyMap(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java
index a89e7e4..9028cb3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/SavepointTriggerRequestBodyTest.java
@@ -19,6 +19,7 @@
 package org.apache.flink.runtime.rest.messages.job.savepoints;
 
 import org.apache.flink.runtime.rest.messages.RestRequestMarshallingTestBase;
+import org.apache.flink.runtime.rest.messages.TriggerId;
 
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -44,8 +45,10 @@ public class SavepointTriggerRequestBodyTest
     public static Collection<Object[]> data() {
         return Arrays.asList(
                 new Object[][] {
-                    {new SavepointTriggerRequestBody("/tmp", true)},
-                    {new SavepointTriggerRequestBody("/tmp", false)}
+                    {new SavepointTriggerRequestBody("/tmp", true, null)},
+                    {new SavepointTriggerRequestBody("/tmp", false, null)},
+                    {new SavepointTriggerRequestBody("/tmp", true, new TriggerId())},
+                    {new SavepointTriggerRequestBody("/tmp", false, new TriggerId())},
                 });
     }
 
@@ -63,5 +66,6 @@ public class SavepointTriggerRequestBodyTest
     protected void assertOriginalEqualsToUnmarshalled(
             final SavepointTriggerRequestBody expected, final SavepointTriggerRequestBody actual) {
         assertEquals(expected.getTargetDirectory(), actual.getTargetDirectory());
+        assertEquals(expected.getTriggerId(), actual.getTriggerId());
     }
 }

[flink] 04/04: [FLINK-24208][rest] Add StopWithSavepointTriggerRequestBodyTest

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 3a561323f7a81990a4c239a886f8c6c404d92f0a
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Nov 4 09:47:45 2021 +0100

    [FLINK-24208][rest] Add StopWithSavepointTriggerRequestBodyTest
---
 .../StopWithSavepointTriggerRequestBodyTest.java   | 74 ++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/StopWithSavepointTriggerRequestBodyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/StopWithSavepointTriggerRequestBodyTest.java
new file mode 100644
index 0000000..7d45c63
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/savepoints/StopWithSavepointTriggerRequestBodyTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.rest.messages.job.savepoints;
+
+import org.apache.flink.runtime.rest.messages.RestRequestMarshallingTestBase;
+import org.apache.flink.runtime.rest.messages.TriggerId;
+import org.apache.flink.runtime.rest.messages.job.savepoints.stop.StopWithSavepointRequestBody;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static org.junit.Assert.assertEquals;
+
+/** Tests for {@link StopWithSavepointRequestBody}. */
+@RunWith(Parameterized.class)
+public class StopWithSavepointTriggerRequestBodyTest
+        extends RestRequestMarshallingTestBase<StopWithSavepointRequestBody> {
+
+    private final StopWithSavepointRequestBody savepointTriggerRequestBody;
+
+    public StopWithSavepointTriggerRequestBodyTest(
+            final StopWithSavepointRequestBody savepointTriggerRequestBody) {
+        this.savepointTriggerRequestBody = savepointTriggerRequestBody;
+    }
+
+    @Parameterized.Parameters
+    public static Collection<Object[]> data() {
+        return Arrays.asList(
+                new Object[][] {
+                    {new StopWithSavepointRequestBody("/tmp", true, null)},
+                    {new StopWithSavepointRequestBody("/tmp", false, null)},
+                    {new StopWithSavepointRequestBody("/tmp", true, new TriggerId())},
+                    {new StopWithSavepointRequestBody("/tmp", false, new TriggerId())},
+                });
+    }
+
+    @Override
+    protected Class<StopWithSavepointRequestBody> getTestRequestClass() {
+        return StopWithSavepointRequestBody.class;
+    }
+
+    @Override
+    protected StopWithSavepointRequestBody getTestRequestInstance() {
+        return savepointTriggerRequestBody;
+    }
+
+    @Override
+    protected void assertOriginalEqualsToUnmarshalled(
+            final StopWithSavepointRequestBody expected,
+            final StopWithSavepointRequestBody actual) {
+        assertEquals(expected.getTargetDirectory(), actual.getTargetDirectory());
+        assertEquals(expected.getTriggerId(), actual.getTriggerId());
+        assertEquals(expected.shouldDrain(), actual.shouldDrain());
+    }
+}

[flink] 01/04: [hotfix] Fix formatting issues in generated files

Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 95eca7e0bc4e368eab58ed4c527d2ad2d8061962
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Wed Oct 27 12:01:09 2021 +0200

    [hotfix] Fix formatting issues in generated files
---
 docs/layouts/shortcodes/generated/rest_v1_dispatcher.html | 6 +++---
 flink-runtime-web/src/test/resources/rest_api_v1.snapshot | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
index 47428aa..bcb1b44 100644
--- a/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
+++ b/docs/layouts/shortcodes/generated/rest_v1_dispatcher.html
@@ -2202,6 +2202,9 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
     "checkpoint_storage" : {
       "type" : "string"
     },
+    "checkpoints_after_tasks_finish" : {
+      "type" : "boolean"
+    },
     "externalization" : {
       "type" : "object",
       "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:checkpoints:CheckpointConfigInfo:ExternalizedCheckpointInfo",
@@ -2237,9 +2240,6 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa
     },
     "unaligned_checkpoints" : {
       "type" : "boolean"
-    },
-    "checkpoints_after_tasks_finish" : {
-      "type" : "boolean"
     }
   }
 }            </code>
diff --git a/flink-runtime-web/src/test/resources/rest_api_v1.snapshot b/flink-runtime-web/src/test/resources/rest_api_v1.snapshot
index c384e7d..083c081 100644
--- a/flink-runtime-web/src/test/resources/rest_api_v1.snapshot
+++ b/flink-runtime-web/src/test/resources/rest_api_v1.snapshot
@@ -1277,7 +1277,7 @@
           "type" : "integer"
         },
         "checkpoints_after_tasks_finish" : {
-           "type" : "boolean"
+          "type" : "boolean"
         }
       }
     }