You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2021/08/23 08:44:57 UTC
[flink] branch master updated: [FLINK-23871][Runtime/Coordination]
Dispatcher should handle finishing job exception when recover
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new ee7cdcd [FLINK-23871][Runtime/Coordination] Dispatcher should handle finishing job exception when recover
ee7cdcd is described below
commit ee7cdcd1174bd83c495280c35ca7ded769a8f186
Author: Aitozi <yu...@alibaba-inc.com>
AuthorDate: Thu Aug 19 17:44:01 2021 +0800
[FLINK-23871][Runtime/Coordination] Dispatcher should handle finishing job exception when recover
This closes #16894.
---
.../runtime/jobmanager/OnCompletionActions.java | 3 --
.../jobmaster/DefaultJobMasterServiceProcess.java | 9 -----
.../runtime/jobmaster/JobAlreadyDoneException.java | 38 ++++++++++++++++++++++
.../JobMasterServiceLeadershipRunner.java | 7 +++-
.../DefaultJobMasterServiceProcessTest.java | 14 --------
.../JobMasterServiceLeadershipRunnerTest.java | 25 ++++++++++++++
.../runtime/jobmaster/utils/JobMasterBuilder.java | 5 ---
7 files changed, 69 insertions(+), 32 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
index f6d820f..40811d5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
@@ -31,9 +31,6 @@ public interface OnCompletionActions {
*/
void jobReachedGloballyTerminalState(ExecutionGraphInfo executionGraphInfo);
- /** Job was finished by another JobMaster. */
- void jobFinishedByOther();
-
/** The {@link JobMaster} failed while executing the job. */
void jobMasterFailed(Throwable cause);
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java
index b087ecd..5ee7a30 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java
@@ -215,15 +215,6 @@ public class DefaultJobMasterServiceProcess
}
@Override
- public void jobFinishedByOther() {
- LOG.debug(
- "Job {} under leader id {} has been finished by another process.",
- jobId,
- leaderSessionId);
- resultFuture.completeExceptionally(new JobNotFinishedException(jobId));
- }
-
- @Override
public void jobMasterFailed(Throwable cause) {
LOG.debug("Job {} under leader id {} failed.", jobId, leaderSessionId);
resultFuture.completeExceptionally(cause);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobAlreadyDoneException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobAlreadyDoneException.java
new file mode 100644
index 0000000..8b6cb6b
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobAlreadyDoneException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.JobException;
+
+/** Exception indicating that a Flink job failed in finishing phase. */
+class JobAlreadyDoneException extends JobException {
+ private static final long serialVersionUID = 5906282500322681417L;
+
+ JobAlreadyDoneException(JobID jobId) {
+ super(
+ "The job ("
+ + jobId
+ + ") has already terminated, but we cannot tell the final "
+ + "state and also not serve the job result. This can happen if Flink "
+ + "fails over shortly after finishing a job and before reporting "
+ + "the result. Please check the output of your job to see whether the "
+ + "execution was successful or not.");
+ }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java
index 2e21f6b..45782d4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java
@@ -285,7 +285,12 @@ public class JobMasterServiceLeadershipRunner implements JobManagerRunner, Leade
}
private void jobAlreadyDone() {
- resultFuture.completeExceptionally(new JobNotFinishedException(getJobID()));
+ resultFuture.complete(
+ JobManagerRunnerResult.forSuccess(
+ new ExecutionGraphInfo(
+ jobMasterServiceProcessFactory.createArchivedExecutionGraph(
+ JobStatus.FAILED,
+ new JobAlreadyDoneException(getJobID())))));
}
private RunningJobsRegistry.JobSchedulingStatus getJobSchedulingStatus() throws FlinkException {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java
index f3442ca..cfce046 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java
@@ -199,20 +199,6 @@ public class DefaultJobMasterServiceProcessTest extends TestLogger {
is(JobStatus.FINISHED));
}
- @Test
- public void testJobFinishedByOther() {
- final CompletableFuture<JobMasterService> jobMasterServiceFuture =
- new CompletableFuture<>();
- DefaultJobMasterServiceProcess serviceProcess = createTestInstance(jobMasterServiceFuture);
- jobMasterServiceFuture.complete(new TestingJobMasterService());
-
- serviceProcess.jobFinishedByOther();
-
- assertThat(
- serviceProcess.getResultFuture(),
- futureWillCompleteExceptionally(JobNotFinishedException.class, TIMEOUT));
- }
-
private DefaultJobMasterServiceProcess createTestInstance(
CompletableFuture<JobMasterService> jobMasterServiceFuture) {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java
index 9829cd9..f79893d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.jobmaster;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.JobStatus;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.core.testutils.FlinkMatchers;
@@ -659,6 +660,30 @@ public class JobMasterServiceLeadershipRunnerTest extends TestLogger {
"Result future should be completed exceptionally."));
}
+ @Test
+ public void testJobAlreadyDone() throws Exception {
+ JobID jobID = new JobID();
+ try (JobManagerRunner jobManagerRunner =
+ newJobMasterServiceLeadershipRunnerBuilder()
+ .setJobMasterServiceProcessFactory(
+ TestingJobMasterServiceProcessFactory.newBuilder()
+ .setJobId(jobID)
+ .build())
+ .build()) {
+ runningJobsRegistry.setJobFinished(jobID);
+ jobManagerRunner.start();
+ leaderElectionService.isLeader(UUID.randomUUID());
+
+ final CompletableFuture<JobManagerRunnerResult> resultFuture =
+ jobManagerRunner.getResultFuture();
+
+ JobManagerRunnerResult result = resultFuture.get();
+ assertEquals(
+ JobStatus.FAILED,
+ result.getExecutionGraphInfo().getArchivedExecutionGraph().getState());
+ }
+ }
+
private void assertJobNotFinished(CompletableFuture<JobManagerRunnerResult> resultFuture)
throws ExecutionException, InterruptedException {
final JobManagerRunnerResult jobManagerRunnerResult = resultFuture.get();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java
index c488525..45fc3d0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java
@@ -219,11 +219,6 @@ public class JobMasterBuilder {
}
@Override
- public void jobFinishedByOther() {
- jobFinishedByOtherFuture.complete(null);
- }
-
- @Override
public void jobMasterFailed(Throwable cause) {
jobMasterFailedFuture.complete(cause);
}