You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2021/08/23 08:44:57 UTC

[flink] branch master updated: [FLINK-23871][Runtime/Coordination] Dispatcher should handle finishing job exception when recover

This is an automated email from the ASF dual-hosted git repository.

trohrmann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new ee7cdcd  [FLINK-23871][Runtime/Coordination] Dispatcher should handle finishing job exception when recover
ee7cdcd is described below

commit ee7cdcd1174bd83c495280c35ca7ded769a8f186
Author: Aitozi <yu...@alibaba-inc.com>
AuthorDate: Thu Aug 19 17:44:01 2021 +0800

    [FLINK-23871][Runtime/Coordination] Dispatcher should handle finishing job exception when recover
    
    This closes #16894.
---
 .../runtime/jobmanager/OnCompletionActions.java    |  3 --
 .../jobmaster/DefaultJobMasterServiceProcess.java  |  9 -----
 .../runtime/jobmaster/JobAlreadyDoneException.java | 38 ++++++++++++++++++++++
 .../JobMasterServiceLeadershipRunner.java          |  7 +++-
 .../DefaultJobMasterServiceProcessTest.java        | 14 --------
 .../JobMasterServiceLeadershipRunnerTest.java      | 25 ++++++++++++++
 .../runtime/jobmaster/utils/JobMasterBuilder.java  |  5 ---
 7 files changed, 69 insertions(+), 32 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
index f6d820f..40811d5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/OnCompletionActions.java
@@ -31,9 +31,6 @@ public interface OnCompletionActions {
      */
     void jobReachedGloballyTerminalState(ExecutionGraphInfo executionGraphInfo);
 
-    /** Job was finished by another JobMaster. */
-    void jobFinishedByOther();
-
     /** The {@link JobMaster} failed while executing the job. */
     void jobMasterFailed(Throwable cause);
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java
index b087ecd..5ee7a30 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcess.java
@@ -215,15 +215,6 @@ public class DefaultJobMasterServiceProcess
     }
 
     @Override
-    public void jobFinishedByOther() {
-        LOG.debug(
-                "Job {} under leader id {} has been finished by another process.",
-                jobId,
-                leaderSessionId);
-        resultFuture.completeExceptionally(new JobNotFinishedException(jobId));
-    }
-
-    @Override
     public void jobMasterFailed(Throwable cause) {
         LOG.debug("Job {} under leader id {} failed.", jobId, leaderSessionId);
         resultFuture.completeExceptionally(cause);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobAlreadyDoneException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobAlreadyDoneException.java
new file mode 100644
index 0000000..8b6cb6b
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobAlreadyDoneException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobmaster;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.JobException;
+
+/** Exception indicating that a Flink job failed in finishing phase. */
+class JobAlreadyDoneException extends JobException {
+    private static final long serialVersionUID = 5906282500322681417L;
+
+    JobAlreadyDoneException(JobID jobId) {
+        super(
+                "The job ("
+                        + jobId
+                        + ") has already terminated, but we cannot tell the final "
+                        + "state and also not serve the job result. This can happen if Flink "
+                        + "fails over shortly after finishing a job and before reporting "
+                        + "the result. Please check the output of your job to see whether the "
+                        + "execution was successful or not.");
+    }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java
index 2e21f6b..45782d4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunner.java
@@ -285,7 +285,12 @@ public class JobMasterServiceLeadershipRunner implements JobManagerRunner, Leade
     }
 
     private void jobAlreadyDone() {
-        resultFuture.completeExceptionally(new JobNotFinishedException(getJobID()));
+        resultFuture.complete(
+                JobManagerRunnerResult.forSuccess(
+                        new ExecutionGraphInfo(
+                                jobMasterServiceProcessFactory.createArchivedExecutionGraph(
+                                        JobStatus.FAILED,
+                                        new JobAlreadyDoneException(getJobID())))));
     }
 
     private RunningJobsRegistry.JobSchedulingStatus getJobSchedulingStatus() throws FlinkException {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java
index f3442ca..cfce046 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/DefaultJobMasterServiceProcessTest.java
@@ -199,20 +199,6 @@ public class DefaultJobMasterServiceProcessTest extends TestLogger {
                 is(JobStatus.FINISHED));
     }
 
-    @Test
-    public void testJobFinishedByOther() {
-        final CompletableFuture<JobMasterService> jobMasterServiceFuture =
-                new CompletableFuture<>();
-        DefaultJobMasterServiceProcess serviceProcess = createTestInstance(jobMasterServiceFuture);
-        jobMasterServiceFuture.complete(new TestingJobMasterService());
-
-        serviceProcess.jobFinishedByOther();
-
-        assertThat(
-                serviceProcess.getResultFuture(),
-                futureWillCompleteExceptionally(JobNotFinishedException.class, TIMEOUT));
-    }
-
     private DefaultJobMasterServiceProcess createTestInstance(
             CompletableFuture<JobMasterService> jobMasterServiceFuture) {
 
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java
index 9829cd9..f79893d 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobMasterServiceLeadershipRunnerTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.runtime.jobmaster;
 
+import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobStatus;
 import org.apache.flink.api.common.time.Time;
 import org.apache.flink.core.testutils.FlinkMatchers;
@@ -659,6 +660,30 @@ public class JobMasterServiceLeadershipRunnerTest extends TestLogger {
                         "Result future should be completed exceptionally."));
     }
 
+    @Test
+    public void testJobAlreadyDone() throws Exception {
+        JobID jobID = new JobID();
+        try (JobManagerRunner jobManagerRunner =
+                newJobMasterServiceLeadershipRunnerBuilder()
+                        .setJobMasterServiceProcessFactory(
+                                TestingJobMasterServiceProcessFactory.newBuilder()
+                                        .setJobId(jobID)
+                                        .build())
+                        .build()) {
+            runningJobsRegistry.setJobFinished(jobID);
+            jobManagerRunner.start();
+            leaderElectionService.isLeader(UUID.randomUUID());
+
+            final CompletableFuture<JobManagerRunnerResult> resultFuture =
+                    jobManagerRunner.getResultFuture();
+
+            JobManagerRunnerResult result = resultFuture.get();
+            assertEquals(
+                    JobStatus.FAILED,
+                    result.getExecutionGraphInfo().getArchivedExecutionGraph().getState());
+        }
+    }
+
     private void assertJobNotFinished(CompletableFuture<JobManagerRunnerResult> resultFuture)
             throws ExecutionException, InterruptedException {
         final JobManagerRunnerResult jobManagerRunnerResult = resultFuture.get();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java
index c488525..45fc3d0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/utils/JobMasterBuilder.java
@@ -219,11 +219,6 @@ public class JobMasterBuilder {
         }
 
         @Override
-        public void jobFinishedByOther() {
-            jobFinishedByOtherFuture.complete(null);
-        }
-
-        @Override
         public void jobMasterFailed(Throwable cause) {
             jobMasterFailedFuture.complete(cause);
         }