You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ti...@apache.org on 2019/11/29 13:53:30 UTC
[flink] 02/09: [FLINK-14762][client] ClusterClient#submitJob
returns CompletableFuture of JobID
This is an automated email from the ASF dual-hosted git repository.
tison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 1266c88dc28393d1497c1ba37def8c3acdd6ddba
Author: tison <wa...@gmail.com>
AuthorDate: Fri Nov 29 09:45:16 2019 +0800
[FLINK-14762][client] ClusterClient#submitJob returns CompletableFuture of JobID
---
.../src/main/java/org/apache/flink/client/ClientUtils.java | 2 --
.../client/deployment/AbstractSessionClusterExecutor.java | 2 --
.../java/org/apache/flink/client/program/ClusterClient.java | 8 +++-----
.../org/apache/flink/client/program/MiniClusterClient.java | 4 ++--
.../apache/flink/client/program/rest/RestClusterClient.java | 12 ++----------
.../apache/flink/client/program/TestingClusterClient.java | 3 +--
.../environment/RemoteStreamExecutionEnvironmentTest.java | 5 ++---
.../apache/flink/test/runtime/FileBufferReaderITCase.java | 7 +++----
.../java/org/apache/flink/test/runtime/SchedulingITCase.java | 7 +++----
.../org/apache/flink/yarn/YARNHighAvailabilityITCase.java | 7 +------
10 files changed, 17 insertions(+), 40 deletions(-)
diff --git a/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java b/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java
index 70d9ddb..675fe84 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/ClientUtils.java
@@ -108,7 +108,6 @@ public enum ClientUtils {
try {
return client
.submitJob(jobGraph)
- .thenApply(JobSubmissionResult::getJobID)
.thenApply(DetachedJobExecutionResult::new)
.get();
} catch (InterruptedException | ExecutionException e) {
@@ -130,7 +129,6 @@ public enum ClientUtils {
try {
jobResult = client
.submitJob(jobGraph)
- .thenApply(JobSubmissionResult::getJobID)
.thenCompose(client::requestJobResult)
.get();
} catch (InterruptedException | ExecutionException e) {
diff --git a/flink-clients/src/main/java/org/apache/flink/client/deployment/AbstractSessionClusterExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/deployment/AbstractSessionClusterExecutor.java
index a1a1cb2..93b815a 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/deployment/AbstractSessionClusterExecutor.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/deployment/AbstractSessionClusterExecutor.java
@@ -19,7 +19,6 @@
package org.apache.flink.client.deployment;
import org.apache.flink.annotation.Internal;
-import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.api.dag.Pipeline;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.configuration.Configuration;
@@ -60,7 +59,6 @@ public class AbstractSessionClusterExecutor<ClusterID, ClientFactory extends Clu
final ClusterClient<ClusterID> clusterClient = clusterDescriptor.retrieve(clusterID);
return clusterClient
.submitJob(jobGraph)
- .thenApply(JobSubmissionResult::getJobID)
.thenApply(jobID -> new ClusterClientJobClientAdapter<ClusterID>(clusterClient, jobID) {
@Override
protected void doClose() {
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
index eadcbe7..ad35526 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
@@ -19,7 +19,6 @@
package org.apache.flink.client.program;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.client.JobStatusMessage;
import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -29,7 +28,6 @@ import org.apache.flink.runtime.messages.Acknowledge;
import org.apache.flink.util.FlinkException;
import org.apache.flink.util.OptionalFailure;
-import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.Collection;
@@ -92,9 +90,9 @@ public interface ClusterClient<T> extends AutoCloseable {
* Submit the given {@link JobGraph} to the cluster.
*
* @param jobGraph to submit
- * @return Future which is completed with the {@link JobSubmissionResult}
+ * @return {@link JobID} of the submitted job
*/
- CompletableFuture<JobSubmissionResult> submitJob(@Nonnull JobGraph jobGraph);
+ CompletableFuture<JobID> submitJob(JobGraph jobGraph);
/**
* Requests the {@link JobStatus} of the job with the given {@link JobID}.
@@ -107,7 +105,7 @@ public interface ClusterClient<T> extends AutoCloseable {
* @param jobId for which to request the {@link JobResult}
* @return Future which is completed with the {@link JobResult}
*/
- CompletableFuture<JobResult> requestJobResult(@Nonnull JobID jobId);
+ CompletableFuture<JobResult> requestJobResult(JobID jobId);
/**
* Requests and returns the accumulators for the given job identifier. Accumulators can be
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java
index ff0ad52..1f32c0a 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java
@@ -66,8 +66,8 @@ public class MiniClusterClient implements ClusterClient<MiniClusterClient.MiniCl
}
@Override
- public CompletableFuture<JobSubmissionResult> submitJob(@Nonnull JobGraph jobGraph) {
- return miniCluster.submitJob(jobGraph);
+ public CompletableFuture<JobID> submitJob(@Nonnull JobGraph jobGraph) {
+ return miniCluster.submitJob(jobGraph).thenApply(JobSubmissionResult::getJobID);
}
@Override
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
index 40c2b64..bacbe15 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
@@ -20,7 +20,6 @@ package org.apache.flink.client.program.rest;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.api.common.accumulators.AccumulatorHelper;
import org.apache.flink.api.common.cache.DistributedCache;
import org.apache.flink.api.common.time.Time;
@@ -263,14 +262,8 @@ public class RestClusterClient<T> implements ClusterClient<T> {
});
}
- /**
- * Submits the given {@link JobGraph} to the dispatcher.
- *
- * @param jobGraph to submit
- * @return Future which is completed with the submission response
- */
@Override
- public CompletableFuture<JobSubmissionResult> submitJob(@Nonnull JobGraph jobGraph) {
+ public CompletableFuture<JobID> submitJob(@Nonnull JobGraph jobGraph) {
CompletableFuture<java.nio.file.Path> jobGraphFileFuture = CompletableFuture.supplyAsync(() -> {
try {
final java.nio.file.Path jobGraphFile = Files.createTempFile("flink-jobgraph", ".bin");
@@ -328,8 +321,7 @@ public class RestClusterClient<T> implements ClusterClient<T> {
});
return submissionFuture
- .thenApply(
- (JobSubmitResponseBody jobSubmitResponseBody) -> new JobSubmissionResult(jobGraph.getJobID()))
+ .thenApply(ignore -> jobGraph.getJobID())
.exceptionally(
(Throwable throwable) -> {
throw new CompletionException(new JobSubmissionException(jobGraph.getJobID(), "Failed to submit JobGraph.", ExceptionUtils.stripCompletionException(throwable)));
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/TestingClusterClient.java b/flink-clients/src/test/java/org/apache/flink/client/program/TestingClusterClient.java
index b13ab94..a711c55 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/TestingClusterClient.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/TestingClusterClient.java
@@ -19,7 +19,6 @@
package org.apache.flink.client.program;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.client.JobStatusMessage;
import org.apache.flink.runtime.jobgraph.JobGraph;
@@ -95,7 +94,7 @@ public class TestingClusterClient<T> implements ClusterClient<T> {
}
@Override
- public CompletableFuture<JobSubmissionResult> submitJob(@Nonnull JobGraph jobGraph) {
+ public CompletableFuture<JobID> submitJob(@Nonnull JobGraph jobGraph) {
throw new UnsupportedOperationException();
}
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java
index 319e981..4e3e5ef 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.api.environment;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.client.RemoteExecutor;
import org.apache.flink.client.program.rest.RestClusterClient;
import org.apache.flink.configuration.Configuration;
@@ -68,7 +67,7 @@ public class RemoteStreamExecutionEnvironmentTest extends TestLogger {
.build();
RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class);
- when(mockedClient.submitJob(any())).thenReturn(CompletableFuture.completedFuture(new JobSubmissionResult(jobID)));
+ when(mockedClient.submitJob(any())).thenReturn(CompletableFuture.completedFuture(jobID));
when(mockedClient.requestJobResult(any())).thenReturn(CompletableFuture.completedFuture(jobResult));
PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenAnswer((invocation) -> {
@@ -106,7 +105,7 @@ public class RemoteStreamExecutionEnvironmentTest extends TestLogger {
RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class);
PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenReturn(mockedClient);
- when(mockedClient.submitJob(any())).thenReturn(CompletableFuture.completedFuture(new JobSubmissionResult(jobID)));
+ when(mockedClient.submitJob(any())).thenReturn(CompletableFuture.completedFuture(jobID));
when(mockedClient.requestJobResult(eq(jobID))).thenReturn(CompletableFuture.completedFuture(jobResult));
JobExecutionResult actualResult = env.execute("fakeJobName");
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java
index 4cd2daa..9449fc1 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java
@@ -18,7 +18,7 @@
package org.apache.flink.test.runtime;
-import org.apache.flink.api.common.JobSubmissionResult;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.client.program.MiniClusterClient;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.NettyShuffleEnvironmentOptions;
@@ -99,11 +99,10 @@ public class FileBufferReaderITCase extends TestLogger {
final MiniClusterClient client = new MiniClusterClient(configuration, miniCluster);
final JobGraph jobGraph = createJobGraph();
- final CompletableFuture<JobSubmissionResult> submitFuture = client.submitJob(jobGraph);
// wait for the submission to succeed
- final JobSubmissionResult result = submitFuture.get();
+ final JobID jobID = client.submitJob(jobGraph).get();
- final CompletableFuture<JobResult> resultFuture = client.requestJobResult(result.getJobID());
+ final CompletableFuture<JobResult> resultFuture = client.requestJobResult(jobID);
final JobResult jobResult = resultFuture.get();
assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/SchedulingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/SchedulingITCase.java
index 113d304..218210a 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/runtime/SchedulingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/SchedulingITCase.java
@@ -19,7 +19,7 @@
package org.apache.flink.test.runtime;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.JobSubmissionResult;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.client.program.MiniClusterClient;
import org.apache.flink.configuration.CheckpointingOptions;
@@ -118,12 +118,11 @@ public class SchedulingITCase extends TestLogger {
MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
JobGraph jobGraph = createJobGraph(slotIdleTimeout << 1, parallelism);
- CompletableFuture<JobSubmissionResult> submissionFuture = miniClusterClient.submitJob(jobGraph);
// wait for the submission to succeed
- JobSubmissionResult jobSubmissionResult = submissionFuture.get();
+ JobID jobID = miniClusterClient.submitJob(jobGraph).get();
- CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobSubmissionResult.getJobID());
+ CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobID);
JobResult jobResult = resultFuture.get();
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index a30046e..e7af40b 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -19,7 +19,6 @@
package org.apache.flink.yarn;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.JobSubmissionResult;
import org.apache.flink.api.common.time.Deadline;
import org.apache.flink.client.deployment.ClusterDeploymentException;
import org.apache.flink.client.deployment.ClusterSpecification;
@@ -301,11 +300,7 @@ public class YARNHighAvailabilityITCase extends YarnTestBase {
}
private JobID submitJob(RestClusterClient<ApplicationId> restClusterClient) throws InterruptedException, java.util.concurrent.ExecutionException {
- final CompletableFuture<JobSubmissionResult> jobSubmissionResultCompletableFuture =
- restClusterClient.submitJob(job);
-
- final JobSubmissionResult jobSubmissionResult = jobSubmissionResultCompletableFuture.get();
- return jobSubmissionResult.getJobID();
+ return restClusterClient.submitJob(job).get();
}
private void killApplicationMaster(final String processName) throws IOException, InterruptedException {