You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2019/06/06 08:08:47 UTC
[flink] 03/03: [FLINK-12614][yarn] Refactor test to not do
assertions in @After methods
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 350846507450ba7afa0159a2c574bd2f44bacaac
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Fri May 24 10:45:59 2019 +0200
[FLINK-12614][yarn] Refactor test to not do assertions in @After methods
---
.../flink/yarn/YARNHighAvailabilityITCase.java | 64 ++--
.../java/org/apache/flink/yarn/YARNITCase.java | 96 +++---
.../yarn/YARNSessionCapacitySchedulerITCase.java | 340 +++++++++++----------
.../apache/flink/yarn/YARNSessionFIFOITCase.java | 275 +++++++++--------
.../flink/yarn/YARNSessionFIFOSecuredITCase.java | 61 ++--
.../apache/flink/yarn/YarnConfigurationITCase.java | 178 +++++------
.../java/org/apache/flink/yarn/YarnTestBase.java | 56 ++--
7 files changed, 559 insertions(+), 511 deletions(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index f9a7c8e..8b22b25 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -153,51 +153,55 @@ public class YARNHighAvailabilityITCase extends YarnTestBase {
*/
@Test
public void testKillYarnSessionClusterEntrypoint() throws Exception {
- assumeTrue(
- "This test kills processes via the pkill command. Thus, it only runs on Linux, Mac OS, Free BSD and Solaris.",
- OperatingSystem.isLinux() || OperatingSystem.isMac() || OperatingSystem.isFreeBSD() || OperatingSystem.isSolaris());
+ runTest(() -> {
+ assumeTrue(
+ "This test kills processes via the pkill command. Thus, it only runs on Linux, Mac OS, Free BSD and Solaris.",
+ OperatingSystem.isLinux() || OperatingSystem.isMac() || OperatingSystem.isFreeBSD() || OperatingSystem.isSolaris());
- final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
+ final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
- try {
- final JobID jobId = submitJob(restClusterClient);
- final ApplicationId id = restClusterClient.getClusterId();
+ try {
+ final JobID jobId = submitJob(restClusterClient);
+ final ApplicationId id = restClusterClient.getClusterId();
- waitUntilJobIsRunning(restClusterClient, jobId);
+ waitUntilJobIsRunning(restClusterClient, jobId);
- killApplicationMaster(yarnClusterDescriptor.getYarnSessionClusterEntrypoint());
- waitForApplicationAttempt(id, 2);
+ killApplicationMaster(yarnClusterDescriptor.getYarnSessionClusterEntrypoint());
+ waitForApplicationAttempt(id, 2);
- waitForJobTermination(restClusterClient, jobId);
+ waitForJobTermination(restClusterClient, jobId);
- killApplicationAndWait(id);
- } finally {
- restClusterClient.shutdown();
- }
+ killApplicationAndWait(id);
+ } finally {
+ restClusterClient.shutdown();
+ }
+ });
}
@Test
public void testJobRecoversAfterKillingTaskManager() throws Exception {
- final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ runTest(() -> {
+ final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
- try {
- final JobID jobId = submitJob(restClusterClient);
- waitUntilJobIsRunning(restClusterClient, jobId);
+ final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
+ try {
+ final JobID jobId = submitJob(restClusterClient);
+ waitUntilJobIsRunning(restClusterClient, jobId);
- stopTaskManagerContainer();
- waitUntilJobIsRestarted(restClusterClient, jobId, 1);
+ stopTaskManagerContainer();
+ waitUntilJobIsRestarted(restClusterClient, jobId, 1);
- waitForJobTermination(restClusterClient, jobId);
+ waitForJobTermination(restClusterClient, jobId);
- killApplicationAndWait(restClusterClient.getClusterId());
- } finally {
- restClusterClient.shutdown();
- }
+ killApplicationAndWait(restClusterClient.getClusterId());
+ } finally {
+ restClusterClient.shutdown();
+ }
+ });
}
private void waitForApplicationAttempt(final ApplicationId applicationId, final int attemptId) throws Exception {
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
index 56cb1e9..470c04d 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
@@ -58,69 +58,71 @@ public class YARNITCase extends YarnTestBase {
@Test
public void testPerJobMode() throws Exception {
- Configuration configuration = new Configuration();
- configuration.setString(AkkaOptions.ASK_TIMEOUT, "30 s");
- final YarnClient yarnClient = getYarnClient();
+ runTest(() -> {
+ Configuration configuration = new Configuration();
+ configuration.setString(AkkaOptions.ASK_TIMEOUT, "30 s");
+ final YarnClient yarnClient = getYarnClient();
- try (final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
- configuration,
- getYarnConfiguration(),
- System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR),
- yarnClient,
- true)) {
+ try (final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
+ configuration,
+ getYarnConfiguration(),
+ System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR),
+ yarnClient,
+ true)) {
- yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
- .setMasterMemoryMB(768)
- .setTaskManagerMemoryMB(1024)
- .setSlotsPerTaskManager(1)
- .setNumberTaskManagers(1)
- .createClusterSpecification();
+ final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
+ .setMasterMemoryMB(768)
+ .setTaskManagerMemoryMB(1024)
+ .setSlotsPerTaskManager(1)
+ .setNumberTaskManagers(1)
+ .createClusterSpecification();
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(2);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(2);
- env.addSource(new NoDataSource())
- .shuffle()
- .addSink(new DiscardingSink<>());
+ env.addSource(new NoDataSource())
+ .shuffle()
+ .addSink(new DiscardingSink<>());
- final JobGraph jobGraph = env.getStreamGraph().getJobGraph();
+ final JobGraph jobGraph = env.getStreamGraph().getJobGraph();
- File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));
+ File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));
- jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
+ jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
- ApplicationId applicationId = null;
- ClusterClient<ApplicationId> clusterClient = null;
+ ApplicationId applicationId = null;
+ ClusterClient<ApplicationId> clusterClient = null;
- try {
- clusterClient = yarnClusterDescriptor.deployJobCluster(
- clusterSpecification,
- jobGraph,
- false);
- applicationId = clusterClient.getClusterId();
+ try {
+ clusterClient = yarnClusterDescriptor.deployJobCluster(
+ clusterSpecification,
+ jobGraph,
+ false);
+ applicationId = clusterClient.getClusterId();
- assertThat(clusterClient, is(instanceOf(RestClusterClient.class)));
- final RestClusterClient<ApplicationId> restClusterClient = (RestClusterClient<ApplicationId>) clusterClient;
+ assertThat(clusterClient, is(instanceOf(RestClusterClient.class)));
+ final RestClusterClient<ApplicationId> restClusterClient = (RestClusterClient<ApplicationId>) clusterClient;
- final CompletableFuture<JobResult> jobResultCompletableFuture = restClusterClient.requestJobResult(jobGraph.getJobID());
+ final CompletableFuture<JobResult> jobResultCompletableFuture = restClusterClient.requestJobResult(jobGraph.getJobID());
- final JobResult jobResult = jobResultCompletableFuture.get();
+ final JobResult jobResult = jobResultCompletableFuture.get();
- assertThat(jobResult, is(notNullValue()));
- assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
- } finally {
- if (clusterClient != null) {
- clusterClient.shutdown();
- }
+ assertThat(jobResult, is(notNullValue()));
+ assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
+ } finally {
+ if (clusterClient != null) {
+ clusterClient.shutdown();
+ }
- if (applicationId != null) {
- yarnClusterDescriptor.killCluster(applicationId);
+ if (applicationId != null) {
+ yarnClusterDescriptor.killCluster(applicationId);
+ }
}
}
- }
+ });
}
}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
index 8e724f4..140b8c8 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
@@ -147,14 +147,14 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* can be started from the command line.
*/
@Test
- public void testStartYarnSessionClusterInQaTeamQueue() throws IOException {
- runWithArgs(new String[]{
- "-j", flinkUberjar.getAbsolutePath(),
- "-t", flinkLibFolder.getAbsolutePath(),
- "-t", flinkShadedHadoopDir.getAbsolutePath(),
- "-jm", "768m",
- "-tm", "1024m", "-qu", "qa-team"},
- "Flink JobManager is now running on ", null, RunTypes.YARN_SESSION, 0);
+ public void testStartYarnSessionClusterInQaTeamQueue() throws Exception {
+ runTest(() -> runWithArgs(new String[]{
+ "-j", flinkUberjar.getAbsolutePath(),
+ "-t", flinkLibFolder.getAbsolutePath(),
+ "-t", flinkShadedHadoopDir.getAbsolutePath(),
+ "-jm", "768m",
+ "-tm", "1024m", "-qu", "qa-team"},
+ "Flink JobManager is now running on ", null, RunTypes.YARN_SESSION, 0));
}
/**
@@ -165,25 +165,27 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* The parallelism is requested at the YARN client (-ys).
*/
@Test
- public void perJobYarnCluster() throws IOException {
- LOG.info("Starting perJobYarnCluster()");
- addTestAppender(CliFrontend.class, Level.INFO);
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
- runWithArgs(new String[]{"run", "-m", "yarn-cluster",
- "-yj", flinkUberjar.getAbsolutePath(),
- "-yt", flinkLibFolder.getAbsolutePath(),
- "-yt", flinkShadedHadoopDir.getAbsolutePath(),
- "-yn", "1",
- "-ys", "2", //test that the job is executed with a DOP of 2
- "-yjm", "768m",
- "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
- /* test succeeded after this string */
- "Program execution finished",
- /* prohibited strings: (to verify the parallelism) */
- // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
- new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
- RunTypes.CLI_FRONTEND, 0, true);
- LOG.info("Finished perJobYarnCluster()");
+ public void perJobYarnCluster() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting perJobYarnCluster()");
+ addTestAppender(CliFrontend.class, Level.INFO);
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+ runWithArgs(new String[]{"run", "-m", "yarn-cluster",
+ "-yj", flinkUberjar.getAbsolutePath(),
+ "-yt", flinkLibFolder.getAbsolutePath(),
+ "-yt", flinkShadedHadoopDir.getAbsolutePath(),
+ "-yn", "1",
+ "-ys", "2", //test that the job is executed with a DOP of 2
+ "-yjm", "768m",
+ "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
+ /* test succeeded after this string */
+ "Program execution finished",
+ /* prohibited strings: (to verify the parallelism) */
+ // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
+ new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
+ RunTypes.CLI_FRONTEND, 0, true);
+ LOG.info("Finished perJobYarnCluster()");
+ });
}
/**
@@ -195,41 +197,43 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* memory remains.
*/
@Test
- public void perJobYarnClusterOffHeap() throws IOException {
- LOG.info("Starting perJobYarnCluster()");
- addTestAppender(CliFrontend.class, Level.INFO);
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
-
- // set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
- final long taskManagerMemoryMB = 1024;
- //noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
- final long networkBuffersMB = NetworkEnvironmentConfiguration.calculateNetworkBufferMemory(
- (taskManagerMemoryMB - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
- new Configuration()) >> 20;
- final long offHeapMemory = taskManagerMemoryMB
- - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
- // cutoff memory (will be added automatically)
- - networkBuffersMB // amount of memory used for network buffers
- - 100; // reserve something for the Java heap space
-
- runWithArgs(new String[]{"run", "-m", "yarn-cluster",
- "-yj", flinkUberjar.getAbsolutePath(),
- "-yt", flinkLibFolder.getAbsolutePath(),
- "-yt", flinkShadedHadoopDir.getAbsolutePath(),
- "-yn", "1",
- "-ys", "2", //test that the job is executed with a DOP of 2
- "-yjm", "768m",
- "-ytm", taskManagerMemoryMB + "m",
- "-yD", "taskmanager.memory.off-heap=true",
- "-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
- "-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
- /* test succeeded after this string */
- "Program execution finished",
- /* prohibited strings: (to verify the parallelism) */
- // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
- new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
- RunTypes.CLI_FRONTEND, 0, true);
- LOG.info("Finished perJobYarnCluster()");
+ public void perJobYarnClusterOffHeap() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting perJobYarnCluster()");
+ addTestAppender(CliFrontend.class, Level.INFO);
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+
+ // set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
+ final long taskManagerMemoryMB = 1024;
+ //noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
+ final long networkBuffersMB = NetworkEnvironmentConfiguration.calculateNetworkBufferMemory(
+ (taskManagerMemoryMB - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
+ new Configuration()) >> 20;
+ final long offHeapMemory = taskManagerMemoryMB
+ - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
+ // cutoff memory (will be added automatically)
+ - networkBuffersMB // amount of memory used for network buffers
+ - 100; // reserve something for the Java heap space
+
+ runWithArgs(new String[]{"run", "-m", "yarn-cluster",
+ "-yj", flinkUberjar.getAbsolutePath(),
+ "-yt", flinkLibFolder.getAbsolutePath(),
+ "-yt", flinkShadedHadoopDir.getAbsolutePath(),
+ "-yn", "1",
+ "-ys", "2", //test that the job is executed with a DOP of 2
+ "-yjm", "768m",
+ "-ytm", taskManagerMemoryMB + "m",
+ "-yD", "taskmanager.memory.off-heap=true",
+ "-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
+ "-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
+ /* test succeeded after this string */
+ "Program execution finished",
+ /* prohibited strings: (to verify the parallelism) */
+ // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
+ new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
+ RunTypes.CLI_FRONTEND, 0, true);
+ LOG.info("Finished perJobYarnCluster()");
+ });
}
/**
@@ -248,63 +252,65 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
*/
@Test
public void testVCoresAreSetCorrectlyAndJobManagerHostnameAreShownInWebInterfaceAndDynamicPropertiesAndYarnApplicationNameAndTaskManagerSlots() throws Exception {
- checkForProhibitedLogContents = false;
- final Runner yarnSessionClusterRunner = startWithArgs(new String[]{
- "-j", flinkUberjar.getAbsolutePath(),
- "-t", flinkLibFolder.getAbsolutePath(),
- "-t", flinkShadedHadoopDir.getAbsolutePath(),
- "-jm", "768m",
- "-tm", "1024m",
- "-s", "3", // set the slots 3 to check if the vCores are set properly!
- "-nm", "customName",
- "-Dfancy-configuration-value=veryFancy",
- "-Dyarn.maximum-failed-containers=3",
- "-D" + YarnConfigOptions.VCORES.key() + "=2"},
- "Flink JobManager is now running on ",
- RunTypes.YARN_SESSION);
+ runTest(() -> {
+ checkForProhibitedLogContents = false;
+ final Runner yarnSessionClusterRunner = startWithArgs(new String[]{
+ "-j", flinkUberjar.getAbsolutePath(),
+ "-t", flinkLibFolder.getAbsolutePath(),
+ "-t", flinkShadedHadoopDir.getAbsolutePath(),
+ "-jm", "768m",
+ "-tm", "1024m",
+ "-s", "3", // set the slots 3 to check if the vCores are set properly!
+ "-nm", "customName",
+ "-Dfancy-configuration-value=veryFancy",
+ "-Dyarn.maximum-failed-containers=3",
+ "-D" + YarnConfigOptions.VCORES.key() + "=2"},
+ "Flink JobManager is now running on ",
+ RunTypes.YARN_SESSION);
- try {
- final String logs = outContent.toString();
- final HostAndPort hostAndPort = parseJobManagerHostname(logs);
- final String host = hostAndPort.getHostText();
- final int port = hostAndPort.getPort();
- LOG.info("Extracted hostname:port: {}", host, port);
-
- submitJob("WindowJoin.jar");
-
- //
- // Assert that custom YARN application name "customName" is set
- //
- final ApplicationReport applicationReport = getOnlyApplicationReport();
- assertEquals("customName", applicationReport.getName());
-
- //
- // Assert the number of TaskManager slots are set
- //
- waitForTaskManagerRegistration(host, port, Duration.ofMillis(30_000));
- assertNumberOfSlotsPerTask(host, port, 3);
-
- final Map<String, String> flinkConfig = getFlinkConfig(host, port);
-
- //
- // Assert dynamic properties
- //
- assertThat(flinkConfig, hasEntry("fancy-configuration-value", "veryFancy"));
- assertThat(flinkConfig, hasEntry("yarn.maximum-failed-containers", "3"));
-
- //
- // FLINK-2213: assert that vcores are set
- //
- assertThat(flinkConfig, hasEntry(YarnConfigOptions.VCORES.key(), "2"));
-
- //
- // FLINK-1902: check if jobmanager hostname is shown in web interface
- //
- assertThat(flinkConfig, hasEntry(JobManagerOptions.ADDRESS.key(), host));
- } finally {
- yarnSessionClusterRunner.sendStop();
- yarnSessionClusterRunner.join();
- }
+ try {
+ final String logs = outContent.toString();
+ final HostAndPort hostAndPort = parseJobManagerHostname(logs);
+ final String host = hostAndPort.getHostText();
+ final int port = hostAndPort.getPort();
+ LOG.info("Extracted hostname:port: {}", host, port);
+
+ submitJob("WindowJoin.jar");
+
+ //
+ // Assert that custom YARN application name "customName" is set
+ //
+ final ApplicationReport applicationReport = getOnlyApplicationReport();
+ assertEquals("customName", applicationReport.getName());
+
+ //
+ // Assert the number of TaskManager slots are set
+ //
+ waitForTaskManagerRegistration(host, port, Duration.ofMillis(30_000));
+ assertNumberOfSlotsPerTask(host, port, 3);
+
+ final Map<String, String> flinkConfig = getFlinkConfig(host, port);
+
+ //
+ // Assert dynamic properties
+ //
+ assertThat(flinkConfig, hasEntry("fancy-configuration-value", "veryFancy"));
+ assertThat(flinkConfig, hasEntry("yarn.maximum-failed-containers", "3"));
+
+ //
+ // FLINK-2213: assert that vcores are set
+ //
+ assertThat(flinkConfig, hasEntry(YarnConfigOptions.VCORES.key(), "2"));
+
+ //
+ // FLINK-1902: check if jobmanager hostname is shown in web interface
+ //
+ assertThat(flinkConfig, hasEntry(JobManagerOptions.ADDRESS.key(), host));
+ } finally {
+ yarnSessionClusterRunner.sendStop();
+ yarnSessionClusterRunner.join();
+ }
+ });
}
private static HostAndPort parseJobManagerHostname(final String logs) {
@@ -398,50 +404,54 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* target queue. With an error message, we can help users identifying the issue)
*/
@Test
- public void testNonexistingQueueWARNmessage() throws IOException {
- LOG.info("Starting testNonexistingQueueWARNmessage()");
- addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
- try {
- runWithArgs(new String[]{"-j", flinkUberjar.getAbsolutePath(),
- "-t", flinkLibFolder.getAbsolutePath(),
- "-t", flinkShadedHadoopDir.getAbsolutePath(),
- "-n", "1",
- "-jm", "768m",
- "-tm", "1024m",
- "-qu", "doesntExist"}, "to unknown queue: doesntExist", null, RunTypes.YARN_SESSION, 1);
- } catch (Exception e) {
- assertTrue(ExceptionUtils.findThrowableWithMessage(e, "to unknown queue: doesntExist").isPresent());
- }
- checkForLogString("The specified queue 'doesntExist' does not exist. Available queues");
- LOG.info("Finished testNonexistingQueueWARNmessage()");
+ public void testNonexistingQueueWARNmessage() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting testNonexistingQueueWARNmessage()");
+ addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
+ try {
+ runWithArgs(new String[]{"-j", flinkUberjar.getAbsolutePath(),
+ "-t", flinkLibFolder.getAbsolutePath(),
+ "-t", flinkShadedHadoopDir.getAbsolutePath(),
+ "-n", "1",
+ "-jm", "768m",
+ "-tm", "1024m",
+ "-qu", "doesntExist"}, "to unknown queue: doesntExist", null, RunTypes.YARN_SESSION, 1);
+ } catch (Exception e) {
+ assertTrue(ExceptionUtils.findThrowableWithMessage(e, "to unknown queue: doesntExist").isPresent());
+ }
+ checkForLogString("The specified queue 'doesntExist' does not exist. Available queues");
+ LOG.info("Finished testNonexistingQueueWARNmessage()");
+ });
}
/**
* Test per-job yarn cluster with the parallelism set at the CliFrontend instead of the YARN client.
*/
@Test
- public void perJobYarnClusterWithParallelism() throws IOException {
- LOG.info("Starting perJobYarnClusterWithParallelism()");
- // write log messages to stdout as well, so that the runWithArgs() method
- // is catching the log output
- addTestAppender(CliFrontend.class, Level.INFO);
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
- runWithArgs(new String[]{"run",
- "-p", "2", //test that the job is executed with a DOP of 2
- "-m", "yarn-cluster",
- "-yj", flinkUberjar.getAbsolutePath(),
- "-yt", flinkLibFolder.getAbsolutePath(),
- "-yt", flinkShadedHadoopDir.getAbsolutePath(),
- "-yn", "1",
- "-ys", "2",
- "-yjm", "768m",
- "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
+ public void perJobYarnClusterWithParallelism() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting perJobYarnClusterWithParallelism()");
+ // write log messages to stdout as well, so that the runWithArgs() method
+ // is catching the log output
+ addTestAppender(CliFrontend.class, Level.INFO);
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+ runWithArgs(new String[]{"run",
+ "-p", "2", //test that the job is executed with a DOP of 2
+ "-m", "yarn-cluster",
+ "-yj", flinkUberjar.getAbsolutePath(),
+ "-yt", flinkLibFolder.getAbsolutePath(),
+ "-yt", flinkShadedHadoopDir.getAbsolutePath(),
+ "-yn", "1",
+ "-ys", "2",
+ "-yjm", "768m",
+ "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
/* test succeeded after this string */
- "Program execution finished",
- /* prohibited strings: (we want to see "DataSink (...) (2/2) switched to FINISHED") */
- new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
- RunTypes.CLI_FRONTEND, 0, true);
- LOG.info("Finished perJobYarnClusterWithParallelism()");
+ "Program execution finished",
+ /* prohibited strings: (we want to see "DataSink (...) (2/2) switched to FINISHED") */
+ new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
+ RunTypes.CLI_FRONTEND, 0, true);
+ LOG.info("Finished perJobYarnClusterWithParallelism()");
+ });
}
/**
@@ -449,13 +459,15 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
*/
@Test(timeout = 60000)
public void testDetachedPerJobYarnCluster() throws Exception {
- LOG.info("Starting testDetachedPerJobYarnCluster()");
+ runTest(() -> {
+ LOG.info("Starting testDetachedPerJobYarnCluster()");
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
- testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
+ testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
- LOG.info("Finished testDetachedPerJobYarnCluster()");
+ LOG.info("Finished testDetachedPerJobYarnCluster()");
+ });
}
/**
@@ -463,13 +475,15 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
*/
@Test(timeout = 60000)
public void testDetachedPerJobYarnClusterWithStreamingJob() throws Exception {
- LOG.info("Starting testDetachedPerJobYarnClusterWithStreamingJob()");
+ runTest(() -> {
+ LOG.info("Starting testDetachedPerJobYarnClusterWithStreamingJob()");
- File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
+ File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
- testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
+ testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
- LOG.info("Finished testDetachedPerJobYarnClusterWithStreamingJob()");
+ LOG.info("Finished testDetachedPerJobYarnClusterWithStreamingJob()");
+ });
}
private void testDetachedPerJobYarnClusterInternal(String job) throws Exception {
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index 0fdce25..34ed63c 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -44,7 +44,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -78,133 +77,139 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
ensureNoProhibitedStringInLogFiles(PROHIBITED_STRINGS, WHITELISTED_STRINGS);
}
+ @Test(timeout = 60000)
+ public void testDetachedMode() throws Exception {
+ runTest(this::runDetachedModeTest);
+ }
+
/**
* Test regular operation, including command line parameter parsing.
*/
- @Test(timeout = 60000) // timeout after a minute.
- public void testDetachedMode() throws InterruptedException, IOException {
- LOG.info("Starting testDetachedMode()");
- addTestAppender(FlinkYarnSessionCli.class, Level.INFO);
-
- File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
- // get temporary file for reading input data for wordcount example
- File tmpInFile = tmp.newFile();
- FileUtils.writeStringToFile(tmpInFile, WordCountData.TEXT);
-
- ArrayList<String> args = new ArrayList<>();
- args.add("-j");
- args.add(flinkUberjar.getAbsolutePath());
-
- args.add("-t");
- args.add(flinkLibFolder.getAbsolutePath());
-
- args.add("-t");
- args.add(flinkShadedHadoopDir.getAbsolutePath());
-
- args.add("-n");
- args.add("1");
-
- args.add("-jm");
- args.add("768m");
-
- args.add("-tm");
- args.add("1024m");
-
- if (SecureTestEnvironment.getTestKeytab() != null) {
- args.add("-D" + SecurityOptions.KERBEROS_LOGIN_KEYTAB.key() + "=" + SecureTestEnvironment.getTestKeytab());
- }
- if (SecureTestEnvironment.getHadoopServicePrincipal() != null) {
- args.add("-D" + SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key() + "=" + SecureTestEnvironment.getHadoopServicePrincipal());
- }
-
- args.add("--name");
- args.add("MyCustomName");
-
- args.add("--applicationType");
- args.add("Apache Flink 1.x");
-
- args.add("--detached");
-
- Runner clusterRunner =
- startWithArgs(
- args.toArray(new String[args.size()]),
- "Flink JobManager is now running on", RunTypes.YARN_SESSION);
-
- // before checking any strings outputted by the CLI, first give it time to return
- clusterRunner.join();
-
- // actually run a program, otherwise we wouldn't necessarily see any TaskManagers
- // be brought up
- Runner jobRunner = startWithArgs(new String[]{"run",
- "--detached", exampleJarLocation.getAbsolutePath(),
- "--input", tmpInFile.getAbsoluteFile().toString()},
- "Job has been submitted with JobID", RunTypes.CLI_FRONTEND);
-
- jobRunner.join();
-
- // in "new" mode we can only wait after the job is submitted, because TMs
- // are spun up lazily
- LOG.info("Waiting until two containers are running");
- // wait until two containers are running
- while (getRunningContainers() < 2) {
- sleep(500);
- }
-
- // make sure we have two TMs running in either mode
- long startTime = System.nanoTime();
- while (System.nanoTime() - startTime < TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS) &&
- !(verifyStringsInNamedLogFiles(
- new String[]{"switched from state RUNNING to FINISHED"}, "jobmanager.log"))) {
- LOG.info("Still waiting for cluster to finish job...");
- sleep(500);
- }
-
- LOG.info("Two containers are running. Killing the application");
-
- // kill application "externally".
- try {
- YarnClient yc = YarnClient.createYarnClient();
- yc.init(YARN_CONFIGURATION);
- yc.start();
- List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
- Assert.assertEquals(1, apps.size()); // Only one running
- ApplicationReport app = apps.get(0);
-
- Assert.assertEquals("MyCustomName", app.getName());
- Assert.assertEquals("Apache Flink 1.x", app.getApplicationType());
- ApplicationId id = app.getApplicationId();
- yc.killApplication(id);
-
- while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0 &&
- yc.getApplications(EnumSet.of(YarnApplicationState.FINISHED)).size() == 0) {
+ void runDetachedModeTest() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting testDetachedMode()");
+ addTestAppender(FlinkYarnSessionCli.class, Level.INFO);
+
+ File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
+ // get temporary file for reading input data for wordcount example
+ File tmpInFile = tmp.newFile();
+ FileUtils.writeStringToFile(tmpInFile, WordCountData.TEXT);
+
+ ArrayList<String> args = new ArrayList<>();
+ args.add("-j");
+ args.add(flinkUberjar.getAbsolutePath());
+
+ args.add("-t");
+ args.add(flinkLibFolder.getAbsolutePath());
+
+ args.add("-t");
+ args.add(flinkShadedHadoopDir.getAbsolutePath());
+
+ args.add("-n");
+ args.add("1");
+
+ args.add("-jm");
+ args.add("768m");
+
+ args.add("-tm");
+ args.add("1024m");
+
+ if (SecureTestEnvironment.getTestKeytab() != null) {
+ args.add("-D" + SecurityOptions.KERBEROS_LOGIN_KEYTAB.key() + "=" + SecureTestEnvironment.getTestKeytab());
+ }
+ if (SecureTestEnvironment.getHadoopServicePrincipal() != null) {
+ args.add("-D" + SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key() + "=" + SecureTestEnvironment.getHadoopServicePrincipal());
+ }
+
+ args.add("--name");
+ args.add("MyCustomName");
+
+ args.add("--applicationType");
+ args.add("Apache Flink 1.x");
+
+ args.add("--detached");
+
+ Runner clusterRunner =
+ startWithArgs(
+ args.toArray(new String[args.size()]),
+ "Flink JobManager is now running on", RunTypes.YARN_SESSION);
+
+ // before checking any strings outputted by the CLI, first give it time to return
+ clusterRunner.join();
+
+ // actually run a program, otherwise we wouldn't necessarily see any TaskManagers
+ // be brought up
+ Runner jobRunner = startWithArgs(new String[]{"run",
+ "--detached", exampleJarLocation.getAbsolutePath(),
+ "--input", tmpInFile.getAbsoluteFile().toString()},
+ "Job has been submitted with JobID", RunTypes.CLI_FRONTEND);
+
+ jobRunner.join();
+
+ // in "new" mode we can only wait after the job is submitted, because TMs
+ // are spun up lazily
+ LOG.info("Waiting until two containers are running");
+ // wait until two containers are running
+ while (getRunningContainers() < 2) {
sleep(500);
}
- } catch (Throwable t) {
- LOG.warn("Killing failed", t);
- Assert.fail();
- } finally {
- //cleanup the yarn-properties file
- String confDirPath = System.getenv("FLINK_CONF_DIR");
- File configDirectory = new File(confDirPath);
- LOG.info("testDetachedPerJobYarnClusterInternal: Using configuration directory " + configDirectory.getAbsolutePath());
+ // make sure we have two TMs running in either mode
+ long startTime = System.nanoTime();
+ while (System.nanoTime() - startTime < TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS) &&
+ !(verifyStringsInNamedLogFiles(
+ new String[]{"switched from state RUNNING to FINISHED"}, "jobmanager.log"))) {
+ LOG.info("Still waiting for cluster to finish job...");
+ sleep(500);
+ }
- // load the configuration
- LOG.info("testDetachedPerJobYarnClusterInternal: Trying to load configuration file");
- Configuration configuration = GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());
+ LOG.info("Two containers are running. Killing the application");
+ // kill application "externally".
try {
- File yarnPropertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION));
- if (yarnPropertiesFile.exists()) {
- LOG.info("testDetachedPerJobYarnClusterInternal: Cleaning up temporary Yarn address reference: {}", yarnPropertiesFile.getAbsolutePath());
- yarnPropertiesFile.delete();
+ YarnClient yc = YarnClient.createYarnClient();
+ yc.init(YARN_CONFIGURATION);
+ yc.start();
+ List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
+ Assert.assertEquals(1, apps.size()); // Only one running
+ ApplicationReport app = apps.get(0);
+
+ Assert.assertEquals("MyCustomName", app.getName());
+ Assert.assertEquals("Apache Flink 1.x", app.getApplicationType());
+ ApplicationId id = app.getApplicationId();
+ yc.killApplication(id);
+
+ while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0 &&
+ yc.getApplications(EnumSet.of(YarnApplicationState.FINISHED)).size() == 0) {
+ sleep(500);
+ }
+ } catch (Throwable t) {
+ LOG.warn("Killing failed", t);
+ Assert.fail();
+ } finally {
+
+ //cleanup the yarn-properties file
+ String confDirPath = System.getenv("FLINK_CONF_DIR");
+ File configDirectory = new File(confDirPath);
+ LOG.info("testDetachedPerJobYarnClusterInternal: Using configuration directory " + configDirectory.getAbsolutePath());
+
+ // load the configuration
+ LOG.info("testDetachedPerJobYarnClusterInternal: Trying to load configuration file");
+ Configuration configuration = GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());
+
+ try {
+ File yarnPropertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION));
+ if (yarnPropertiesFile.exists()) {
+ LOG.info("testDetachedPerJobYarnClusterInternal: Cleaning up temporary Yarn address reference: {}", yarnPropertiesFile.getAbsolutePath());
+ yarnPropertiesFile.delete();
+ }
+ } catch (Exception e) {
+ LOG.warn("testDetachedPerJobYarnClusterInternal: Exception while deleting the JobManager address file", e);
}
- } catch (Exception e) {
- LOG.warn("testDetachedPerJobYarnClusterInternal: Exception while deleting the JobManager address file", e);
}
- }
- LOG.info("Finished testDetachedMode()");
+ LOG.info("Finished testDetachedMode()");
+ });
}
/**
@@ -213,10 +218,12 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
* <p>This test validates through 666*2 cores in the "cluster".
*/
@Test
- public void testQueryCluster() throws IOException {
- LOG.info("Starting testQueryCluster()");
- runWithArgs(new String[] {"-q"}, "Summary: totalMemory 8192 totalCores 1332", null, RunTypes.YARN_SESSION, 0); // we have 666*2 cores.
- LOG.info("Finished testQueryCluster()");
+ public void testQueryCluster() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting testQueryCluster()");
+ runWithArgs(new String[]{"-q"}, "Summary: totalMemory 8192 totalCores 1332", null, RunTypes.YARN_SESSION, 0); // we have 666*2 cores.
+ LOG.info("Finished testQueryCluster()");
+ });
}
/**
@@ -233,18 +240,20 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
*/
@Ignore("The test is too resource consuming (8.5 GB of memory)")
@Test
- public void testResourceComputation() throws IOException {
- addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
- LOG.info("Starting testResourceComputation()");
- runWithArgs(new String[]{
+ public void testResourceComputation() throws Exception {
+ runTest(() -> {
+ addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
+ LOG.info("Starting testResourceComputation()");
+ runWithArgs(new String[]{
"-j", flinkUberjar.getAbsolutePath(),
"-t", flinkLibFolder.getAbsolutePath(),
"-t", flinkShadedHadoopDir.getAbsolutePath(),
"-n", "5",
"-jm", "256m",
"-tm", "1585m"}, "Number of connected TaskManagers changed to", null, RunTypes.YARN_SESSION, 0);
- LOG.info("Finished testResourceComputation()");
- checkForLogString("This YARN session requires 8437MB of memory in the cluster. There are currently only 8192MB available.");
+ LOG.info("Finished testResourceComputation()");
+ checkForLogString("This YARN session requires 8437MB of memory in the cluster. There are currently only 8192MB available.");
+ });
}
/**
@@ -264,18 +273,20 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
*/
@Ignore("The test is too resource consuming (8 GB of memory)")
@Test
- public void testfullAlloc() throws IOException {
- addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
- LOG.info("Starting testfullAlloc()");
- runWithArgs(new String[]{
+ public void testfullAlloc() throws Exception {
+ runTest(() -> {
+ addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
+ LOG.info("Starting testfullAlloc()");
+ runWithArgs(new String[]{
"-j", flinkUberjar.getAbsolutePath(),
"-t", flinkLibFolder.getAbsolutePath(),
"-t", flinkShadedHadoopDir.getAbsolutePath(),
"-n", "2",
"-jm", "256m",
"-tm", "3840m"}, "Number of connected TaskManagers changed to", null, RunTypes.YARN_SESSION, 0);
- LOG.info("Finished testfullAlloc()");
- checkForLogString("There is not enough memory available in the YARN cluster. The TaskManager(s) require 3840MB each. NodeManagers available: [4096, 4096]\n" +
+ LOG.info("Finished testfullAlloc()");
+ checkForLogString("There is not enough memory available in the YARN cluster. The TaskManager(s) require 3840MB each. NodeManagers available: [4096, 4096]\n" +
"After allocating the JobManager (512MB) and (1/2) TaskManagers, the following NodeManagers are available: [3584, 256]");
+ });
}
}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
index 54a5532..3faf777 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
@@ -40,7 +40,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
@@ -100,41 +99,43 @@ public class YARNSessionFIFOSecuredITCase extends YARNSessionFIFOITCase {
}
@AfterClass
- public static void teardownSecureCluster() throws Exception {
+ public static void teardownSecureCluster() {
LOG.info("tearing down secure cluster environment");
SecureTestEnvironment.cleanup();
}
@Test(timeout = 60000) // timeout after a minute.
@Override
- public void testDetachedMode() throws InterruptedException, IOException {
- super.testDetachedMode();
- final String[] mustHave = {"Login successful for user", "using keytab file"};
- final boolean jobManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
- mustHave,
- "jobmanager.log");
- final boolean taskManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
- mustHave, "taskmanager.log");
-
- Assert.assertThat(
- "The JobManager and the TaskManager should both run with Kerberos.",
- jobManagerRunsWithKerberos && taskManagerRunsWithKerberos,
- Matchers.is(true));
-
- final List<String> amRMTokens = Lists.newArrayList(AMRMTokenIdentifier.KIND_NAME.toString());
- final String jobmanagerContainerId = getContainerIdByLogName("jobmanager.log");
- final String taskmanagerContainerId = getContainerIdByLogName("taskmanager.log");
- final boolean jobmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, jobmanagerContainerId);
- final boolean taskmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, taskmanagerContainerId);
-
- Assert.assertThat(
- "The JobManager should have AMRMToken.",
- jobmanagerWithAmRmToken,
- Matchers.is(true));
- Assert.assertThat(
- "The TaskManager should not have AMRMToken.",
- taskmanagerWithAmRmToken,
- Matchers.is(false));
+ public void testDetachedMode() throws Exception {
+ runTest(() -> {
+ runDetachedModeTest();
+ final String[] mustHave = {"Login successful for user", "using keytab file"};
+ final boolean jobManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
+ mustHave,
+ "jobmanager.log");
+ final boolean taskManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
+ mustHave, "taskmanager.log");
+
+ Assert.assertThat(
+ "The JobManager and the TaskManager should both run with Kerberos.",
+ jobManagerRunsWithKerberos && taskManagerRunsWithKerberos,
+ Matchers.is(true));
+
+ final List<String> amRMTokens = Lists.newArrayList(AMRMTokenIdentifier.KIND_NAME.toString());
+ final String jobmanagerContainerId = getContainerIdByLogName("jobmanager.log");
+ final String taskmanagerContainerId = getContainerIdByLogName("taskmanager.log");
+ final boolean jobmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, jobmanagerContainerId);
+ final boolean taskmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, taskmanagerContainerId);
+
+ Assert.assertThat(
+ "The JobManager should have AMRMToken.",
+ jobmanagerWithAmRmToken,
+ Matchers.is(true));
+ Assert.assertThat(
+ "The TaskManager should not have AMRMToken.",
+ taskmanagerWithAmRmToken,
+ Matchers.is(false));
+ });
}
/* For secure cluster testing, it is enough to run only one test and override below test methods
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
index acca620..28653b6 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
@@ -78,124 +78,126 @@ public class YarnConfigurationITCase extends YarnTestBase {
*/
@Test(timeout = 60000)
public void testFlinkContainerMemory() throws Exception {
- final YarnClient yarnClient = getYarnClient();
- final Configuration configuration = new Configuration(flinkConfiguration);
+ runTest(() -> {
+ final YarnClient yarnClient = getYarnClient();
+ final Configuration configuration = new Configuration(flinkConfiguration);
- final int masterMemory = 64;
- final int taskManagerMemory = 128;
- final int slotsPerTaskManager = 3;
+ final int masterMemory = 64;
+ final int taskManagerMemory = 128;
+ final int slotsPerTaskManager = 3;
- // disable heap cutoff min
- configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);
- configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(1L << 20));
- configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(4L << 20));
+ // disable heap cutoff min
+ configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);
+ configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(1L << 20));
+ configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(4L << 20));
- final YarnConfiguration yarnConfiguration = getYarnConfiguration();
- final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
- configuration,
- yarnConfiguration,
- CliFrontend.getConfigurationDirectoryFromEnv(),
- yarnClient,
- true);
+ final YarnConfiguration yarnConfiguration = getYarnConfiguration();
+ final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
+ configuration,
+ yarnConfiguration,
+ CliFrontend.getConfigurationDirectoryFromEnv(),
+ yarnClient,
+ true);
- clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
- clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
- clusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
+ clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
+ clusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final File streamingWordCountFile = getTestJarPath("WindowJoin.jar");
+ final File streamingWordCountFile = getTestJarPath("WindowJoin.jar");
- final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile);
- final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1);
+ final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile);
+ final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1);
- try {
- final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
- .setMasterMemoryMB(masterMemory)
- .setTaskManagerMemoryMB(taskManagerMemory)
- .setSlotsPerTaskManager(slotsPerTaskManager)
- .createClusterSpecification();
+ try {
+ final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
+ .setMasterMemoryMB(masterMemory)
+ .setTaskManagerMemoryMB(taskManagerMemory)
+ .setSlotsPerTaskManager(slotsPerTaskManager)
+ .createClusterSpecification();
- final ClusterClient<ApplicationId> clusterClient = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true);
+ final ClusterClient<ApplicationId> clusterClient = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true);
- final ApplicationId clusterId = clusterClient.getClusterId();
+ final ApplicationId clusterId = clusterClient.getClusterId();
- final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration), TestingUtils.defaultExecutor());
+ final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration), TestingUtils.defaultExecutor());
- try {
- final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId);
+ try {
+ final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId);
- final ApplicationAttemptId currentApplicationAttemptId = applicationReport.getCurrentApplicationAttemptId();
+ final ApplicationAttemptId currentApplicationAttemptId = applicationReport.getCurrentApplicationAttemptId();
- // wait until we have second container allocated
- List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId);
+ // wait until we have second container allocated
+ List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId);
- while (containers.size() < 2) {
- // this is nasty but Yarn does not offer a better way to wait
- Thread.sleep(50L);
- containers = yarnClient.getContainers(currentApplicationAttemptId);
- }
+ while (containers.size() < 2) {
+ // this is nasty but Yarn does not offer a better way to wait
+ Thread.sleep(50L);
+ containers = yarnClient.getContainers(currentApplicationAttemptId);
+ }
- for (ContainerReport container : containers) {
- if (container.getContainerId().getId() == 1) {
- // this should be the application master
- assertThat(container.getAllocatedResource().getMemory(), is(masterMemory));
- } else {
- assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory));
+ for (ContainerReport container : containers) {
+ if (container.getContainerId().getId() == 1) {
+ // this should be the application master
+ assertThat(container.getAllocatedResource().getMemory(), is(masterMemory));
+ } else {
+ assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory));
+ }
}
- }
- final URI webURI = new URI(clusterClient.getWebInterfaceURL());
+ final URI webURI = new URI(clusterClient.getWebInterfaceURL());
- CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture;
- Collection<TaskManagerInfo> taskManagerInfos;
+ CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture;
+ Collection<TaskManagerInfo> taskManagerInfos;
- while (true) {
- taskManagersInfoCompletableFuture = restClient.sendRequest(
- webURI.getHost(),
- webURI.getPort(),
- TaskManagersHeaders.getInstance(),
- EmptyMessageParameters.getInstance(),
- EmptyRequestBody.getInstance());
+ while (true) {
+ taskManagersInfoCompletableFuture = restClient.sendRequest(
+ webURI.getHost(),
+ webURI.getPort(),
+ TaskManagersHeaders.getInstance(),
+ EmptyMessageParameters.getInstance(),
+ EmptyRequestBody.getInstance());
- final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get();
+ final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get();
- taskManagerInfos = taskManagersInfo.getTaskManagerInfos();
+ taskManagerInfos = taskManagersInfo.getTaskManagerInfos();
- // wait until the task manager has registered and reported its slots
- if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) {
- break;
- } else {
- Thread.sleep(100L);
+ // wait until the task manager has registered and reported its slots
+ if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) {
+ break;
+ } else {
+ Thread.sleep(100L);
+ }
}
- }
- // there should be at least one TaskManagerInfo
- final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
+ // there should be at least one TaskManagerInfo
+ final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
- assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager));
+ assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager));
- final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters.create(
- configuration,
- taskManagerMemory,
- slotsPerTaskManager);
+ final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters.create(
+ configuration,
+ taskManagerMemory,
+ slotsPerTaskManager);
- final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L;
+ final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L;
- // We compare here physical memory assigned to a container with the heap memory that we should pass to
- // jvm as Xmx parameter. Those value might differ significantly due to system page size or jvm
- // implementation therefore we use 15% threshold here.
- assertThat(
- (double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize,
- is(closeTo(1.0, 0.15)));
- } finally {
- restClient.shutdown(TIMEOUT);
- clusterClient.shutdown();
- }
+ // We compare here physical memory assigned to a container with the heap memory that we should pass to
+ // jvm as Xmx parameter. Those value might differ significantly due to system page size or jvm
+ // implementation therefore we use 15% threshold here.
+ assertThat(
+ (double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize,
+ is(closeTo(1.0, 0.15)));
+ } finally {
+ restClient.shutdown(TIMEOUT);
+ clusterClient.shutdown();
+ }
- clusterDescriptor.killCluster(clusterId);
+ clusterDescriptor.killCluster(clusterId);
- } finally {
- clusterDescriptor.close();
- }
+ } finally {
+ clusterDescriptor.close();
+ }
+ });
}
private boolean hasTaskManagerConnectedAndReportedSlots(Collection<TaskManagerInfo> taskManagerInfos) {
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
index 9beaf49..42e3b43 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.runtime.clusterframework.BootstrapTools;
import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.TestLogger;
+import org.apache.flink.util.function.RunnableWithException;
import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
import org.apache.commons.io.FileUtils;
@@ -42,7 +43,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -200,7 +200,7 @@ public abstract class YarnTestBase extends TestLogger {
}
@Before
- public void checkClusterEmpty() {
+ public void setupYarnClient() {
if (yarnClient == null) {
yarnClient = YarnClient.createYarnClient();
yarnClient.init(getYarnConfiguration());
@@ -214,29 +214,43 @@ public abstract class YarnTestBase extends TestLogger {
* Sleep a bit between the tests (we are re-using the YARN cluster for the tests).
*/
@After
- public void sleep() throws IOException, YarnException {
- Deadline deadline = Deadline.now().plus(Duration.ofSeconds(10));
+ public void shutdownYarnClient() {
+ yarnClient.stop();
+ }
- boolean isAnyJobRunning = yarnClient.getApplications().stream()
- .anyMatch(YarnTestBase::isApplicationRunning);
+ protected void runTest(RunnableWithException test) throws Exception {
+ // wrapping the cleanup logic in an AutoClosable automatically suppresses additional exceptions
+ try (final CleanupYarnApplication ignored = new CleanupYarnApplication()) {
+ test.run();
+ }
+ }
- while (deadline.hasTimeLeft() && isAnyJobRunning) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- Assert.fail("Should not happen");
- }
- isAnyJobRunning = yarnClient.getApplications().stream()
+ private class CleanupYarnApplication implements AutoCloseable {
+ @Override
+ public void close() throws Exception {
+ Deadline deadline = Deadline.now().plus(Duration.ofSeconds(10));
+
+ boolean isAnyJobRunning = yarnClient.getApplications().stream()
.anyMatch(YarnTestBase::isApplicationRunning);
- }
- if (isAnyJobRunning) {
- final List<String> runningApps = yarnClient.getApplications().stream()
- .filter(YarnTestBase::isApplicationRunning)
- .map(app -> "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState() + '.')
- .collect(Collectors.toList());
- if (!runningApps.isEmpty()) {
- Assert.fail("There is at least one application on the cluster that is not finished." + runningApps);
+ while (deadline.hasTimeLeft() && isAnyJobRunning) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ Assert.fail("Should not happen");
+ }
+ isAnyJobRunning = yarnClient.getApplications().stream()
+ .anyMatch(YarnTestBase::isApplicationRunning);
+ }
+
+ if (isAnyJobRunning) {
+ final List<String> runningApps = yarnClient.getApplications().stream()
+ .filter(YarnTestBase::isApplicationRunning)
+ .map(app -> "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState() + '.')
+ .collect(Collectors.toList());
+ if (!runningApps.isEmpty()) {
+ Assert.fail("There is at least one application on the cluster that is not finished." + runningApps);
+ }
}
}
}