You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2021/06/03 07:29:45 UTC

[flink] branch release-1.12 updated: [FLINK-22820] Allow storing SUSPENDED jobs in FileArchivedExecutionGraphStore

This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch release-1.12
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.12 by this push:
     new 31f4099  [FLINK-22820] Allow storing SUSPENDED jobs in FileArchivedExecutionGraphStore
31f4099 is described below

commit 31f40993c6c31c6ec26976235feed2c0a43d4cab
Author: Fabian Paul <fa...@ververica.com>
AuthorDate: Tue Jun 1 15:00:16 2021 +0200

    [FLINK-22820] Allow storing SUSPENDED jobs in FileArchivedExecutionGraphStore
---
 .../FileArchivedExecutionGraphStore.java           |   8 +-
 .../FileArchivedExecutionGraphStoreTest.java       | 120 +++++++++++++++++----
 2 files changed, 105 insertions(+), 23 deletions(-)

diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java
index f08c996..920227c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java
@@ -163,12 +163,12 @@ public class FileArchivedExecutionGraphStore implements ArchivedExecutionGraphSt
         final String jobName = archivedExecutionGraph.getJobName();
 
         Preconditions.checkArgument(
-                jobStatus.isGloballyTerminalState(),
+                jobStatus.isTerminalState(),
                 "The job "
                         + jobName
                         + '('
                         + jobId
-                        + ") is not in a globally terminal state. Instead it is in state "
+                        + ") is not in a terminal state. Instead it is in state "
                         + jobStatus
                         + '.');
 
@@ -182,13 +182,15 @@ public class FileArchivedExecutionGraphStore implements ArchivedExecutionGraphSt
             case FAILED:
                 numFailedJobs++;
                 break;
+            case SUSPENDED:
+                break;
             default:
                 throw new IllegalStateException(
                         "The job "
                                 + jobName
                                 + '('
                                 + jobId
-                                + ") should have been in a globally terminal state. "
+                                + ") should have been in a known terminal state. "
                                 + "Instead it was in state "
                                 + jobStatus
                                 + '.');
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java
index 42e0325..47567ce 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java
@@ -21,13 +21,28 @@ package org.apache.flink.runtime.dispatcher;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.JobStatus;
 import org.apache.flink.api.common.time.Time;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.blob.BlobServer;
 import org.apache.flink.runtime.concurrent.ManuallyTriggeredScheduledExecutor;
+import org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory;
+import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponent;
+import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponentFactory;
 import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph;
+import org.apache.flink.runtime.heartbeat.HeartbeatServices;
+import org.apache.flink.runtime.highavailability.HighAvailabilityServices;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.utils.JobGraphTestUtils;
 import org.apache.flink.runtime.messages.webmonitor.JobDetails;
 import org.apache.flink.runtime.messages.webmonitor.JobsOverview;
+import org.apache.flink.runtime.metrics.MetricRegistry;
+import org.apache.flink.runtime.minicluster.MiniCluster;
+import org.apache.flink.runtime.minicluster.MiniClusterConfiguration;
+import org.apache.flink.runtime.resourcemanager.StandaloneResourceManagerFactory;
 import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder;
+import org.apache.flink.runtime.rpc.FatalErrorHandler;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
 import org.apache.flink.runtime.util.ManualTicker;
+import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever;
 import org.apache.flink.util.Preconditions;
 import org.apache.flink.util.TestLogger;
 
@@ -47,6 +62,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.ThreadLocalRandom;
@@ -73,27 +89,13 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger {
      */
     @Test
     public void testPut() throws IOException {
-        final ArchivedExecutionGraph dummyExecutionGraph =
-                new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).build();
-        final File rootDir = temporaryFolder.newFolder();
-
-        try (final FileArchivedExecutionGraphStore executionGraphStore =
-                createDefaultExecutionGraphStore(rootDir)) {
-
-            final File storageDirectory = executionGraphStore.getStorageDir();
-
-            // check that the storage directory is empty
-            assertThat(storageDirectory.listFiles().length, Matchers.equalTo(0));
-
-            executionGraphStore.put(dummyExecutionGraph);
-
-            // check that we have persisted the given execution graph
-            assertThat(storageDirectory.listFiles().length, Matchers.equalTo(1));
+        assertPutJobGraphWithStatus(JobStatus.FINISHED);
+    }
 
-            assertThat(
-                    executionGraphStore.get(dummyExecutionGraph.getJobID()),
-                    new PartialArchivedExecutionGraphMatcher(dummyExecutionGraph));
-        }
+    /** Tests that a SUSPENDED job can be persisted. */
+    @Test
+    public void testPutSuspendedJob() throws IOException {
+        assertPutJobGraphWithStatus(JobStatus.SUSPENDED);
     }
 
     /** Tests that null is returned if we request an unknown JobID. */
@@ -322,6 +324,84 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger {
         }
     }
 
+    /** Tests that a session cluster can terminate gracefully when jobs are still running. */
+    @Test
+    public void testPutSuspendedJobOnClusterShutdown() throws Exception {
+        try (final MiniCluster miniCluster =
+                new PersistingMiniCluster(new MiniClusterConfiguration.Builder().build())) {
+            miniCluster.start();
+            final JobGraph jobGraph = JobGraphTestUtils.createSingleVertexJobGraph();
+            miniCluster.submitJob(jobGraph);
+        }
+    }
+
+    private class PersistingMiniCluster extends MiniCluster {
+
+        PersistingMiniCluster(MiniClusterConfiguration miniClusterConfiguration) {
+            super(miniClusterConfiguration);
+        }
+
+        @Override
+        protected Collection<? extends DispatcherResourceManagerComponent>
+                createDispatcherResourceManagerComponents(
+                        Configuration configuration,
+                        RpcServiceFactory rpcServiceFactory,
+                        HighAvailabilityServices haServices,
+                        BlobServer blobServer,
+                        HeartbeatServices heartbeatServices,
+                        MetricRegistry metricRegistry,
+                        MetricQueryServiceRetriever metricQueryServiceRetriever,
+                        FatalErrorHandler fatalErrorHandler)
+                        throws Exception {
+            final DispatcherResourceManagerComponentFactory
+                    dispatcherResourceManagerComponentFactory =
+                            DefaultDispatcherResourceManagerComponentFactory
+                                    .createSessionComponentFactory(
+                                            StandaloneResourceManagerFactory.getInstance());
+
+            final File rootDir = temporaryFolder.newFolder();
+            final ArchivedExecutionGraphStore executionGraphInfoStore =
+                    createDefaultExecutionGraphStore(rootDir);
+
+            return Collections.singleton(
+                    dispatcherResourceManagerComponentFactory.create(
+                            configuration,
+                            getIOExecutor(),
+                            rpcServiceFactory.createRpcService(),
+                            haServices,
+                            blobServer,
+                            heartbeatServices,
+                            metricRegistry,
+                            executionGraphInfoStore,
+                            metricQueryServiceRetriever,
+                            fatalErrorHandler));
+        }
+    }
+
+    private void assertPutJobGraphWithStatus(JobStatus jobStatus) throws IOException {
+        final ArchivedExecutionGraph dummyExecutionGraph =
+                new ArchivedExecutionGraphBuilder().setState(jobStatus).build();
+        final File rootDir = temporaryFolder.newFolder();
+
+        try (final FileArchivedExecutionGraphStore executionGraphStore =
+                createDefaultExecutionGraphStore(rootDir)) {
+
+            final File storageDirectory = executionGraphStore.getStorageDir();
+
+            // check that the storage directory is empty
+            assertThat(storageDirectory.listFiles().length, Matchers.equalTo(0));
+
+            executionGraphStore.put(dummyExecutionGraph);
+
+            // check that we have persisted the given execution graph
+            assertThat(storageDirectory.listFiles().length, Matchers.equalTo(1));
+
+            assertThat(
+                    executionGraphStore.get(dummyExecutionGraph.getJobID()),
+                    new PartialArchivedExecutionGraphMatcher(dummyExecutionGraph));
+        }
+    }
+
     private Collection<ArchivedExecutionGraph> generateTerminalExecutionGraphs(int number) {
         final Collection<ArchivedExecutionGraph> executionGraphs = new ArrayList<>(number);