You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2019/06/06 08:08:44 UTC
[flink] branch master updated (13a0900 -> 3508465)
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 13a0900 [FLINK-12734][table-planner-blink] remove getVolcanoPlanner method from FlinkOptimizeContext and make RelNodeBlock not depend on TableEnvironment
new 42c8aa5 [hotfix][yarn][tests] Remove unused field
new e1a4db2 [FLINK-12445][yarn] Disable test timeout
new 3508465 [FLINK-12614][yarn] Refactor test to not do assertions in @After methods
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../flink/yarn/YARNHighAvailabilityITCase.java | 64 ++--
.../java/org/apache/flink/yarn/YARNITCase.java | 96 +++---
.../yarn/YARNSessionCapacitySchedulerITCase.java | 342 +++++++++++----------
.../apache/flink/yarn/YARNSessionFIFOITCase.java | 275 +++++++++--------
.../flink/yarn/YARNSessionFIFOSecuredITCase.java | 61 ++--
.../apache/flink/yarn/YarnConfigurationITCase.java | 178 +++++------
.../java/org/apache/flink/yarn/YarnTestBase.java | 58 ++--
7 files changed, 560 insertions(+), 514 deletions(-)
[flink] 03/03: [FLINK-12614][yarn] Refactor test to not do
assertions in @After methods
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 350846507450ba7afa0159a2c574bd2f44bacaac
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Fri May 24 10:45:59 2019 +0200
[FLINK-12614][yarn] Refactor test to not do assertions in @After methods
---
.../flink/yarn/YARNHighAvailabilityITCase.java | 64 ++--
.../java/org/apache/flink/yarn/YARNITCase.java | 96 +++---
.../yarn/YARNSessionCapacitySchedulerITCase.java | 340 +++++++++++----------
.../apache/flink/yarn/YARNSessionFIFOITCase.java | 275 +++++++++--------
.../flink/yarn/YARNSessionFIFOSecuredITCase.java | 61 ++--
.../apache/flink/yarn/YarnConfigurationITCase.java | 178 +++++------
.../java/org/apache/flink/yarn/YarnTestBase.java | 56 ++--
7 files changed, 559 insertions(+), 511 deletions(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index f9a7c8e..8b22b25 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -153,51 +153,55 @@ public class YARNHighAvailabilityITCase extends YarnTestBase {
*/
@Test
public void testKillYarnSessionClusterEntrypoint() throws Exception {
- assumeTrue(
- "This test kills processes via the pkill command. Thus, it only runs on Linux, Mac OS, Free BSD and Solaris.",
- OperatingSystem.isLinux() || OperatingSystem.isMac() || OperatingSystem.isFreeBSD() || OperatingSystem.isSolaris());
+ runTest(() -> {
+ assumeTrue(
+ "This test kills processes via the pkill command. Thus, it only runs on Linux, Mac OS, Free BSD and Solaris.",
+ OperatingSystem.isLinux() || OperatingSystem.isMac() || OperatingSystem.isFreeBSD() || OperatingSystem.isSolaris());
- final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
+ final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
- try {
- final JobID jobId = submitJob(restClusterClient);
- final ApplicationId id = restClusterClient.getClusterId();
+ try {
+ final JobID jobId = submitJob(restClusterClient);
+ final ApplicationId id = restClusterClient.getClusterId();
- waitUntilJobIsRunning(restClusterClient, jobId);
+ waitUntilJobIsRunning(restClusterClient, jobId);
- killApplicationMaster(yarnClusterDescriptor.getYarnSessionClusterEntrypoint());
- waitForApplicationAttempt(id, 2);
+ killApplicationMaster(yarnClusterDescriptor.getYarnSessionClusterEntrypoint());
+ waitForApplicationAttempt(id, 2);
- waitForJobTermination(restClusterClient, jobId);
+ waitForJobTermination(restClusterClient, jobId);
- killApplicationAndWait(id);
- } finally {
- restClusterClient.shutdown();
- }
+ killApplicationAndWait(id);
+ } finally {
+ restClusterClient.shutdown();
+ }
+ });
}
@Test
public void testJobRecoversAfterKillingTaskManager() throws Exception {
- final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ runTest(() -> {
+ final YarnClusterDescriptor yarnClusterDescriptor = setupYarnClusterDescriptor();
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
- try {
- final JobID jobId = submitJob(restClusterClient);
- waitUntilJobIsRunning(restClusterClient, jobId);
+ final RestClusterClient<ApplicationId> restClusterClient = deploySessionCluster(yarnClusterDescriptor);
+ try {
+ final JobID jobId = submitJob(restClusterClient);
+ waitUntilJobIsRunning(restClusterClient, jobId);
- stopTaskManagerContainer();
- waitUntilJobIsRestarted(restClusterClient, jobId, 1);
+ stopTaskManagerContainer();
+ waitUntilJobIsRestarted(restClusterClient, jobId, 1);
- waitForJobTermination(restClusterClient, jobId);
+ waitForJobTermination(restClusterClient, jobId);
- killApplicationAndWait(restClusterClient.getClusterId());
- } finally {
- restClusterClient.shutdown();
- }
+ killApplicationAndWait(restClusterClient.getClusterId());
+ } finally {
+ restClusterClient.shutdown();
+ }
+ });
}
private void waitForApplicationAttempt(final ApplicationId applicationId, final int attemptId) throws Exception {
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
index 56cb1e9..470c04d 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
@@ -58,69 +58,71 @@ public class YARNITCase extends YarnTestBase {
@Test
public void testPerJobMode() throws Exception {
- Configuration configuration = new Configuration();
- configuration.setString(AkkaOptions.ASK_TIMEOUT, "30 s");
- final YarnClient yarnClient = getYarnClient();
+ runTest(() -> {
+ Configuration configuration = new Configuration();
+ configuration.setString(AkkaOptions.ASK_TIMEOUT, "30 s");
+ final YarnClient yarnClient = getYarnClient();
- try (final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
- configuration,
- getYarnConfiguration(),
- System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR),
- yarnClient,
- true)) {
+ try (final YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
+ configuration,
+ getYarnConfiguration(),
+ System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR),
+ yarnClient,
+ true)) {
- yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
- yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ yarnClusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
+ yarnClusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
- .setMasterMemoryMB(768)
- .setTaskManagerMemoryMB(1024)
- .setSlotsPerTaskManager(1)
- .setNumberTaskManagers(1)
- .createClusterSpecification();
+ final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
+ .setMasterMemoryMB(768)
+ .setTaskManagerMemoryMB(1024)
+ .setSlotsPerTaskManager(1)
+ .setNumberTaskManagers(1)
+ .createClusterSpecification();
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(2);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(2);
- env.addSource(new NoDataSource())
- .shuffle()
- .addSink(new DiscardingSink<>());
+ env.addSource(new NoDataSource())
+ .shuffle()
+ .addSink(new DiscardingSink<>());
- final JobGraph jobGraph = env.getStreamGraph().getJobGraph();
+ final JobGraph jobGraph = env.getStreamGraph().getJobGraph();
- File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));
+ File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));
- jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
+ jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
- ApplicationId applicationId = null;
- ClusterClient<ApplicationId> clusterClient = null;
+ ApplicationId applicationId = null;
+ ClusterClient<ApplicationId> clusterClient = null;
- try {
- clusterClient = yarnClusterDescriptor.deployJobCluster(
- clusterSpecification,
- jobGraph,
- false);
- applicationId = clusterClient.getClusterId();
+ try {
+ clusterClient = yarnClusterDescriptor.deployJobCluster(
+ clusterSpecification,
+ jobGraph,
+ false);
+ applicationId = clusterClient.getClusterId();
- assertThat(clusterClient, is(instanceOf(RestClusterClient.class)));
- final RestClusterClient<ApplicationId> restClusterClient = (RestClusterClient<ApplicationId>) clusterClient;
+ assertThat(clusterClient, is(instanceOf(RestClusterClient.class)));
+ final RestClusterClient<ApplicationId> restClusterClient = (RestClusterClient<ApplicationId>) clusterClient;
- final CompletableFuture<JobResult> jobResultCompletableFuture = restClusterClient.requestJobResult(jobGraph.getJobID());
+ final CompletableFuture<JobResult> jobResultCompletableFuture = restClusterClient.requestJobResult(jobGraph.getJobID());
- final JobResult jobResult = jobResultCompletableFuture.get();
+ final JobResult jobResult = jobResultCompletableFuture.get();
- assertThat(jobResult, is(notNullValue()));
- assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
- } finally {
- if (clusterClient != null) {
- clusterClient.shutdown();
- }
+ assertThat(jobResult, is(notNullValue()));
+ assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
+ } finally {
+ if (clusterClient != null) {
+ clusterClient.shutdown();
+ }
- if (applicationId != null) {
- yarnClusterDescriptor.killCluster(applicationId);
+ if (applicationId != null) {
+ yarnClusterDescriptor.killCluster(applicationId);
+ }
}
}
- }
+ });
}
}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
index 8e724f4..140b8c8 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
@@ -147,14 +147,14 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* can be started from the command line.
*/
@Test
- public void testStartYarnSessionClusterInQaTeamQueue() throws IOException {
- runWithArgs(new String[]{
- "-j", flinkUberjar.getAbsolutePath(),
- "-t", flinkLibFolder.getAbsolutePath(),
- "-t", flinkShadedHadoopDir.getAbsolutePath(),
- "-jm", "768m",
- "-tm", "1024m", "-qu", "qa-team"},
- "Flink JobManager is now running on ", null, RunTypes.YARN_SESSION, 0);
+ public void testStartYarnSessionClusterInQaTeamQueue() throws Exception {
+ runTest(() -> runWithArgs(new String[]{
+ "-j", flinkUberjar.getAbsolutePath(),
+ "-t", flinkLibFolder.getAbsolutePath(),
+ "-t", flinkShadedHadoopDir.getAbsolutePath(),
+ "-jm", "768m",
+ "-tm", "1024m", "-qu", "qa-team"},
+ "Flink JobManager is now running on ", null, RunTypes.YARN_SESSION, 0));
}
/**
@@ -165,25 +165,27 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* The parallelism is requested at the YARN client (-ys).
*/
@Test
- public void perJobYarnCluster() throws IOException {
- LOG.info("Starting perJobYarnCluster()");
- addTestAppender(CliFrontend.class, Level.INFO);
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
- runWithArgs(new String[]{"run", "-m", "yarn-cluster",
- "-yj", flinkUberjar.getAbsolutePath(),
- "-yt", flinkLibFolder.getAbsolutePath(),
- "-yt", flinkShadedHadoopDir.getAbsolutePath(),
- "-yn", "1",
- "-ys", "2", //test that the job is executed with a DOP of 2
- "-yjm", "768m",
- "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
- /* test succeeded after this string */
- "Program execution finished",
- /* prohibited strings: (to verify the parallelism) */
- // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
- new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
- RunTypes.CLI_FRONTEND, 0, true);
- LOG.info("Finished perJobYarnCluster()");
+ public void perJobYarnCluster() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting perJobYarnCluster()");
+ addTestAppender(CliFrontend.class, Level.INFO);
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+ runWithArgs(new String[]{"run", "-m", "yarn-cluster",
+ "-yj", flinkUberjar.getAbsolutePath(),
+ "-yt", flinkLibFolder.getAbsolutePath(),
+ "-yt", flinkShadedHadoopDir.getAbsolutePath(),
+ "-yn", "1",
+ "-ys", "2", //test that the job is executed with a DOP of 2
+ "-yjm", "768m",
+ "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
+ /* test succeeded after this string */
+ "Program execution finished",
+ /* prohibited strings: (to verify the parallelism) */
+ // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
+ new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
+ RunTypes.CLI_FRONTEND, 0, true);
+ LOG.info("Finished perJobYarnCluster()");
+ });
}
/**
@@ -195,41 +197,43 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* memory remains.
*/
@Test
- public void perJobYarnClusterOffHeap() throws IOException {
- LOG.info("Starting perJobYarnCluster()");
- addTestAppender(CliFrontend.class, Level.INFO);
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
-
- // set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
- final long taskManagerMemoryMB = 1024;
- //noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
- final long networkBuffersMB = NetworkEnvironmentConfiguration.calculateNetworkBufferMemory(
- (taskManagerMemoryMB - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
- new Configuration()) >> 20;
- final long offHeapMemory = taskManagerMemoryMB
- - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
- // cutoff memory (will be added automatically)
- - networkBuffersMB // amount of memory used for network buffers
- - 100; // reserve something for the Java heap space
-
- runWithArgs(new String[]{"run", "-m", "yarn-cluster",
- "-yj", flinkUberjar.getAbsolutePath(),
- "-yt", flinkLibFolder.getAbsolutePath(),
- "-yt", flinkShadedHadoopDir.getAbsolutePath(),
- "-yn", "1",
- "-ys", "2", //test that the job is executed with a DOP of 2
- "-yjm", "768m",
- "-ytm", taskManagerMemoryMB + "m",
- "-yD", "taskmanager.memory.off-heap=true",
- "-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
- "-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
- /* test succeeded after this string */
- "Program execution finished",
- /* prohibited strings: (to verify the parallelism) */
- // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
- new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
- RunTypes.CLI_FRONTEND, 0, true);
- LOG.info("Finished perJobYarnCluster()");
+ public void perJobYarnClusterOffHeap() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting perJobYarnCluster()");
+ addTestAppender(CliFrontend.class, Level.INFO);
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+
+ // set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
+ final long taskManagerMemoryMB = 1024;
+ //noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
+ final long networkBuffersMB = NetworkEnvironmentConfiguration.calculateNetworkBufferMemory(
+ (taskManagerMemoryMB - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
+ new Configuration()) >> 20;
+ final long offHeapMemory = taskManagerMemoryMB
+ - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
+ // cutoff memory (will be added automatically)
+ - networkBuffersMB // amount of memory used for network buffers
+ - 100; // reserve something for the Java heap space
+
+ runWithArgs(new String[]{"run", "-m", "yarn-cluster",
+ "-yj", flinkUberjar.getAbsolutePath(),
+ "-yt", flinkLibFolder.getAbsolutePath(),
+ "-yt", flinkShadedHadoopDir.getAbsolutePath(),
+ "-yn", "1",
+ "-ys", "2", //test that the job is executed with a DOP of 2
+ "-yjm", "768m",
+ "-ytm", taskManagerMemoryMB + "m",
+ "-yD", "taskmanager.memory.off-heap=true",
+ "-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
+ "-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
+ /* test succeeded after this string */
+ "Program execution finished",
+ /* prohibited strings: (to verify the parallelism) */
+ // (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
+ new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
+ RunTypes.CLI_FRONTEND, 0, true);
+ LOG.info("Finished perJobYarnCluster()");
+ });
}
/**
@@ -248,63 +252,65 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
*/
@Test
public void testVCoresAreSetCorrectlyAndJobManagerHostnameAreShownInWebInterfaceAndDynamicPropertiesAndYarnApplicationNameAndTaskManagerSlots() throws Exception {
- checkForProhibitedLogContents = false;
- final Runner yarnSessionClusterRunner = startWithArgs(new String[]{
- "-j", flinkUberjar.getAbsolutePath(),
- "-t", flinkLibFolder.getAbsolutePath(),
- "-t", flinkShadedHadoopDir.getAbsolutePath(),
- "-jm", "768m",
- "-tm", "1024m",
- "-s", "3", // set the slots 3 to check if the vCores are set properly!
- "-nm", "customName",
- "-Dfancy-configuration-value=veryFancy",
- "-Dyarn.maximum-failed-containers=3",
- "-D" + YarnConfigOptions.VCORES.key() + "=2"},
- "Flink JobManager is now running on ",
- RunTypes.YARN_SESSION);
+ runTest(() -> {
+ checkForProhibitedLogContents = false;
+ final Runner yarnSessionClusterRunner = startWithArgs(new String[]{
+ "-j", flinkUberjar.getAbsolutePath(),
+ "-t", flinkLibFolder.getAbsolutePath(),
+ "-t", flinkShadedHadoopDir.getAbsolutePath(),
+ "-jm", "768m",
+ "-tm", "1024m",
+ "-s", "3", // set the slots 3 to check if the vCores are set properly!
+ "-nm", "customName",
+ "-Dfancy-configuration-value=veryFancy",
+ "-Dyarn.maximum-failed-containers=3",
+ "-D" + YarnConfigOptions.VCORES.key() + "=2"},
+ "Flink JobManager is now running on ",
+ RunTypes.YARN_SESSION);
- try {
- final String logs = outContent.toString();
- final HostAndPort hostAndPort = parseJobManagerHostname(logs);
- final String host = hostAndPort.getHostText();
- final int port = hostAndPort.getPort();
- LOG.info("Extracted hostname:port: {}", host, port);
-
- submitJob("WindowJoin.jar");
-
- //
- // Assert that custom YARN application name "customName" is set
- //
- final ApplicationReport applicationReport = getOnlyApplicationReport();
- assertEquals("customName", applicationReport.getName());
-
- //
- // Assert the number of TaskManager slots are set
- //
- waitForTaskManagerRegistration(host, port, Duration.ofMillis(30_000));
- assertNumberOfSlotsPerTask(host, port, 3);
-
- final Map<String, String> flinkConfig = getFlinkConfig(host, port);
-
- //
- // Assert dynamic properties
- //
- assertThat(flinkConfig, hasEntry("fancy-configuration-value", "veryFancy"));
- assertThat(flinkConfig, hasEntry("yarn.maximum-failed-containers", "3"));
-
- //
- // FLINK-2213: assert that vcores are set
- //
- assertThat(flinkConfig, hasEntry(YarnConfigOptions.VCORES.key(), "2"));
-
- //
- // FLINK-1902: check if jobmanager hostname is shown in web interface
- //
- assertThat(flinkConfig, hasEntry(JobManagerOptions.ADDRESS.key(), host));
- } finally {
- yarnSessionClusterRunner.sendStop();
- yarnSessionClusterRunner.join();
- }
+ try {
+ final String logs = outContent.toString();
+ final HostAndPort hostAndPort = parseJobManagerHostname(logs);
+ final String host = hostAndPort.getHostText();
+ final int port = hostAndPort.getPort();
+ LOG.info("Extracted hostname:port: {}", host, port);
+
+ submitJob("WindowJoin.jar");
+
+ //
+ // Assert that custom YARN application name "customName" is set
+ //
+ final ApplicationReport applicationReport = getOnlyApplicationReport();
+ assertEquals("customName", applicationReport.getName());
+
+ //
+ // Assert the number of TaskManager slots are set
+ //
+ waitForTaskManagerRegistration(host, port, Duration.ofMillis(30_000));
+ assertNumberOfSlotsPerTask(host, port, 3);
+
+ final Map<String, String> flinkConfig = getFlinkConfig(host, port);
+
+ //
+ // Assert dynamic properties
+ //
+ assertThat(flinkConfig, hasEntry("fancy-configuration-value", "veryFancy"));
+ assertThat(flinkConfig, hasEntry("yarn.maximum-failed-containers", "3"));
+
+ //
+ // FLINK-2213: assert that vcores are set
+ //
+ assertThat(flinkConfig, hasEntry(YarnConfigOptions.VCORES.key(), "2"));
+
+ //
+ // FLINK-1902: check if jobmanager hostname is shown in web interface
+ //
+ assertThat(flinkConfig, hasEntry(JobManagerOptions.ADDRESS.key(), host));
+ } finally {
+ yarnSessionClusterRunner.sendStop();
+ yarnSessionClusterRunner.join();
+ }
+ });
}
private static HostAndPort parseJobManagerHostname(final String logs) {
@@ -398,50 +404,54 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
* target queue. With an error message, we can help users identifying the issue)
*/
@Test
- public void testNonexistingQueueWARNmessage() throws IOException {
- LOG.info("Starting testNonexistingQueueWARNmessage()");
- addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
- try {
- runWithArgs(new String[]{"-j", flinkUberjar.getAbsolutePath(),
- "-t", flinkLibFolder.getAbsolutePath(),
- "-t", flinkShadedHadoopDir.getAbsolutePath(),
- "-n", "1",
- "-jm", "768m",
- "-tm", "1024m",
- "-qu", "doesntExist"}, "to unknown queue: doesntExist", null, RunTypes.YARN_SESSION, 1);
- } catch (Exception e) {
- assertTrue(ExceptionUtils.findThrowableWithMessage(e, "to unknown queue: doesntExist").isPresent());
- }
- checkForLogString("The specified queue 'doesntExist' does not exist. Available queues");
- LOG.info("Finished testNonexistingQueueWARNmessage()");
+ public void testNonexistingQueueWARNmessage() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting testNonexistingQueueWARNmessage()");
+ addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
+ try {
+ runWithArgs(new String[]{"-j", flinkUberjar.getAbsolutePath(),
+ "-t", flinkLibFolder.getAbsolutePath(),
+ "-t", flinkShadedHadoopDir.getAbsolutePath(),
+ "-n", "1",
+ "-jm", "768m",
+ "-tm", "1024m",
+ "-qu", "doesntExist"}, "to unknown queue: doesntExist", null, RunTypes.YARN_SESSION, 1);
+ } catch (Exception e) {
+ assertTrue(ExceptionUtils.findThrowableWithMessage(e, "to unknown queue: doesntExist").isPresent());
+ }
+ checkForLogString("The specified queue 'doesntExist' does not exist. Available queues");
+ LOG.info("Finished testNonexistingQueueWARNmessage()");
+ });
}
/**
* Test per-job yarn cluster with the parallelism set at the CliFrontend instead of the YARN client.
*/
@Test
- public void perJobYarnClusterWithParallelism() throws IOException {
- LOG.info("Starting perJobYarnClusterWithParallelism()");
- // write log messages to stdout as well, so that the runWithArgs() method
- // is catching the log output
- addTestAppender(CliFrontend.class, Level.INFO);
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
- runWithArgs(new String[]{"run",
- "-p", "2", //test that the job is executed with a DOP of 2
- "-m", "yarn-cluster",
- "-yj", flinkUberjar.getAbsolutePath(),
- "-yt", flinkLibFolder.getAbsolutePath(),
- "-yt", flinkShadedHadoopDir.getAbsolutePath(),
- "-yn", "1",
- "-ys", "2",
- "-yjm", "768m",
- "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
+ public void perJobYarnClusterWithParallelism() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting perJobYarnClusterWithParallelism()");
+ // write log messages to stdout as well, so that the runWithArgs() method
+ // is catching the log output
+ addTestAppender(CliFrontend.class, Level.INFO);
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+ runWithArgs(new String[]{"run",
+ "-p", "2", //test that the job is executed with a DOP of 2
+ "-m", "yarn-cluster",
+ "-yj", flinkUberjar.getAbsolutePath(),
+ "-yt", flinkLibFolder.getAbsolutePath(),
+ "-yt", flinkShadedHadoopDir.getAbsolutePath(),
+ "-yn", "1",
+ "-ys", "2",
+ "-yjm", "768m",
+ "-ytm", "1024m", exampleJarLocation.getAbsolutePath()},
/* test succeeded after this string */
- "Program execution finished",
- /* prohibited strings: (we want to see "DataSink (...) (2/2) switched to FINISHED") */
- new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
- RunTypes.CLI_FRONTEND, 0, true);
- LOG.info("Finished perJobYarnClusterWithParallelism()");
+ "Program execution finished",
+ /* prohibited strings: (we want to see "DataSink (...) (2/2) switched to FINISHED") */
+ new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
+ RunTypes.CLI_FRONTEND, 0, true);
+ LOG.info("Finished perJobYarnClusterWithParallelism()");
+ });
}
/**
@@ -449,13 +459,15 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
*/
@Test(timeout = 60000)
public void testDetachedPerJobYarnCluster() throws Exception {
- LOG.info("Starting testDetachedPerJobYarnCluster()");
+ runTest(() -> {
+ LOG.info("Starting testDetachedPerJobYarnCluster()");
- File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
+ File exampleJarLocation = getTestJarPath("BatchWordCount.jar");
- testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
+ testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
- LOG.info("Finished testDetachedPerJobYarnCluster()");
+ LOG.info("Finished testDetachedPerJobYarnCluster()");
+ });
}
/**
@@ -463,13 +475,15 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
*/
@Test(timeout = 60000)
public void testDetachedPerJobYarnClusterWithStreamingJob() throws Exception {
- LOG.info("Starting testDetachedPerJobYarnClusterWithStreamingJob()");
+ runTest(() -> {
+ LOG.info("Starting testDetachedPerJobYarnClusterWithStreamingJob()");
- File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
+ File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
- testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
+ testDetachedPerJobYarnClusterInternal(exampleJarLocation.getAbsolutePath());
- LOG.info("Finished testDetachedPerJobYarnClusterWithStreamingJob()");
+ LOG.info("Finished testDetachedPerJobYarnClusterWithStreamingJob()");
+ });
}
private void testDetachedPerJobYarnClusterInternal(String job) throws Exception {
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index 0fdce25..34ed63c 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -44,7 +44,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -78,133 +77,139 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
ensureNoProhibitedStringInLogFiles(PROHIBITED_STRINGS, WHITELISTED_STRINGS);
}
+ @Test(timeout = 60000)
+ public void testDetachedMode() throws Exception {
+ runTest(this::runDetachedModeTest);
+ }
+
/**
* Test regular operation, including command line parameter parsing.
*/
- @Test(timeout = 60000) // timeout after a minute.
- public void testDetachedMode() throws InterruptedException, IOException {
- LOG.info("Starting testDetachedMode()");
- addTestAppender(FlinkYarnSessionCli.class, Level.INFO);
-
- File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
- // get temporary file for reading input data for wordcount example
- File tmpInFile = tmp.newFile();
- FileUtils.writeStringToFile(tmpInFile, WordCountData.TEXT);
-
- ArrayList<String> args = new ArrayList<>();
- args.add("-j");
- args.add(flinkUberjar.getAbsolutePath());
-
- args.add("-t");
- args.add(flinkLibFolder.getAbsolutePath());
-
- args.add("-t");
- args.add(flinkShadedHadoopDir.getAbsolutePath());
-
- args.add("-n");
- args.add("1");
-
- args.add("-jm");
- args.add("768m");
-
- args.add("-tm");
- args.add("1024m");
-
- if (SecureTestEnvironment.getTestKeytab() != null) {
- args.add("-D" + SecurityOptions.KERBEROS_LOGIN_KEYTAB.key() + "=" + SecureTestEnvironment.getTestKeytab());
- }
- if (SecureTestEnvironment.getHadoopServicePrincipal() != null) {
- args.add("-D" + SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key() + "=" + SecureTestEnvironment.getHadoopServicePrincipal());
- }
-
- args.add("--name");
- args.add("MyCustomName");
-
- args.add("--applicationType");
- args.add("Apache Flink 1.x");
-
- args.add("--detached");
-
- Runner clusterRunner =
- startWithArgs(
- args.toArray(new String[args.size()]),
- "Flink JobManager is now running on", RunTypes.YARN_SESSION);
-
- // before checking any strings outputted by the CLI, first give it time to return
- clusterRunner.join();
-
- // actually run a program, otherwise we wouldn't necessarily see any TaskManagers
- // be brought up
- Runner jobRunner = startWithArgs(new String[]{"run",
- "--detached", exampleJarLocation.getAbsolutePath(),
- "--input", tmpInFile.getAbsoluteFile().toString()},
- "Job has been submitted with JobID", RunTypes.CLI_FRONTEND);
-
- jobRunner.join();
-
- // in "new" mode we can only wait after the job is submitted, because TMs
- // are spun up lazily
- LOG.info("Waiting until two containers are running");
- // wait until two containers are running
- while (getRunningContainers() < 2) {
- sleep(500);
- }
-
- // make sure we have two TMs running in either mode
- long startTime = System.nanoTime();
- while (System.nanoTime() - startTime < TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS) &&
- !(verifyStringsInNamedLogFiles(
- new String[]{"switched from state RUNNING to FINISHED"}, "jobmanager.log"))) {
- LOG.info("Still waiting for cluster to finish job...");
- sleep(500);
- }
-
- LOG.info("Two containers are running. Killing the application");
-
- // kill application "externally".
- try {
- YarnClient yc = YarnClient.createYarnClient();
- yc.init(YARN_CONFIGURATION);
- yc.start();
- List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
- Assert.assertEquals(1, apps.size()); // Only one running
- ApplicationReport app = apps.get(0);
-
- Assert.assertEquals("MyCustomName", app.getName());
- Assert.assertEquals("Apache Flink 1.x", app.getApplicationType());
- ApplicationId id = app.getApplicationId();
- yc.killApplication(id);
-
- while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0 &&
- yc.getApplications(EnumSet.of(YarnApplicationState.FINISHED)).size() == 0) {
+ void runDetachedModeTest() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting testDetachedMode()");
+ addTestAppender(FlinkYarnSessionCli.class, Level.INFO);
+
+ File exampleJarLocation = getTestJarPath("StreamingWordCount.jar");
+ // get temporary file for reading input data for wordcount example
+ File tmpInFile = tmp.newFile();
+ FileUtils.writeStringToFile(tmpInFile, WordCountData.TEXT);
+
+ ArrayList<String> args = new ArrayList<>();
+ args.add("-j");
+ args.add(flinkUberjar.getAbsolutePath());
+
+ args.add("-t");
+ args.add(flinkLibFolder.getAbsolutePath());
+
+ args.add("-t");
+ args.add(flinkShadedHadoopDir.getAbsolutePath());
+
+ args.add("-n");
+ args.add("1");
+
+ args.add("-jm");
+ args.add("768m");
+
+ args.add("-tm");
+ args.add("1024m");
+
+ if (SecureTestEnvironment.getTestKeytab() != null) {
+ args.add("-D" + SecurityOptions.KERBEROS_LOGIN_KEYTAB.key() + "=" + SecureTestEnvironment.getTestKeytab());
+ }
+ if (SecureTestEnvironment.getHadoopServicePrincipal() != null) {
+ args.add("-D" + SecurityOptions.KERBEROS_LOGIN_PRINCIPAL.key() + "=" + SecureTestEnvironment.getHadoopServicePrincipal());
+ }
+
+ args.add("--name");
+ args.add("MyCustomName");
+
+ args.add("--applicationType");
+ args.add("Apache Flink 1.x");
+
+ args.add("--detached");
+
+ Runner clusterRunner =
+ startWithArgs(
+ args.toArray(new String[args.size()]),
+ "Flink JobManager is now running on", RunTypes.YARN_SESSION);
+
+ // before checking any strings outputted by the CLI, first give it time to return
+ clusterRunner.join();
+
+ // actually run a program, otherwise we wouldn't necessarily see any TaskManagers
+ // be brought up
+ Runner jobRunner = startWithArgs(new String[]{"run",
+ "--detached", exampleJarLocation.getAbsolutePath(),
+ "--input", tmpInFile.getAbsoluteFile().toString()},
+ "Job has been submitted with JobID", RunTypes.CLI_FRONTEND);
+
+ jobRunner.join();
+
+ // in "new" mode we can only wait after the job is submitted, because TMs
+ // are spun up lazily
+ LOG.info("Waiting until two containers are running");
+ // wait until two containers are running
+ while (getRunningContainers() < 2) {
sleep(500);
}
- } catch (Throwable t) {
- LOG.warn("Killing failed", t);
- Assert.fail();
- } finally {
- //cleanup the yarn-properties file
- String confDirPath = System.getenv("FLINK_CONF_DIR");
- File configDirectory = new File(confDirPath);
- LOG.info("testDetachedPerJobYarnClusterInternal: Using configuration directory " + configDirectory.getAbsolutePath());
+ // make sure we have two TMs running in either mode
+ long startTime = System.nanoTime();
+ while (System.nanoTime() - startTime < TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS) &&
+ !(verifyStringsInNamedLogFiles(
+ new String[]{"switched from state RUNNING to FINISHED"}, "jobmanager.log"))) {
+ LOG.info("Still waiting for cluster to finish job...");
+ sleep(500);
+ }
- // load the configuration
- LOG.info("testDetachedPerJobYarnClusterInternal: Trying to load configuration file");
- Configuration configuration = GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());
+ LOG.info("Two containers are running. Killing the application");
+ // kill application "externally".
try {
- File yarnPropertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION));
- if (yarnPropertiesFile.exists()) {
- LOG.info("testDetachedPerJobYarnClusterInternal: Cleaning up temporary Yarn address reference: {}", yarnPropertiesFile.getAbsolutePath());
- yarnPropertiesFile.delete();
+ YarnClient yc = YarnClient.createYarnClient();
+ yc.init(YARN_CONFIGURATION);
+ yc.start();
+ List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
+ Assert.assertEquals(1, apps.size()); // Only one running
+ ApplicationReport app = apps.get(0);
+
+ Assert.assertEquals("MyCustomName", app.getName());
+ Assert.assertEquals("Apache Flink 1.x", app.getApplicationType());
+ ApplicationId id = app.getApplicationId();
+ yc.killApplication(id);
+
+ while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0 &&
+ yc.getApplications(EnumSet.of(YarnApplicationState.FINISHED)).size() == 0) {
+ sleep(500);
+ }
+ } catch (Throwable t) {
+ LOG.warn("Killing failed", t);
+ Assert.fail();
+ } finally {
+
+ //cleanup the yarn-properties file
+ String confDirPath = System.getenv("FLINK_CONF_DIR");
+ File configDirectory = new File(confDirPath);
+ LOG.info("testDetachedPerJobYarnClusterInternal: Using configuration directory " + configDirectory.getAbsolutePath());
+
+ // load the configuration
+ LOG.info("testDetachedPerJobYarnClusterInternal: Trying to load configuration file");
+ Configuration configuration = GlobalConfiguration.loadConfiguration(configDirectory.getAbsolutePath());
+
+ try {
+ File yarnPropertiesFile = FlinkYarnSessionCli.getYarnPropertiesLocation(configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION));
+ if (yarnPropertiesFile.exists()) {
+ LOG.info("testDetachedPerJobYarnClusterInternal: Cleaning up temporary Yarn address reference: {}", yarnPropertiesFile.getAbsolutePath());
+ yarnPropertiesFile.delete();
+ }
+ } catch (Exception e) {
+ LOG.warn("testDetachedPerJobYarnClusterInternal: Exception while deleting the JobManager address file", e);
}
- } catch (Exception e) {
- LOG.warn("testDetachedPerJobYarnClusterInternal: Exception while deleting the JobManager address file", e);
}
- }
- LOG.info("Finished testDetachedMode()");
+ LOG.info("Finished testDetachedMode()");
+ });
}
/**
@@ -213,10 +218,12 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
* <p>This test validates through 666*2 cores in the "cluster".
*/
@Test
- public void testQueryCluster() throws IOException {
- LOG.info("Starting testQueryCluster()");
- runWithArgs(new String[] {"-q"}, "Summary: totalMemory 8192 totalCores 1332", null, RunTypes.YARN_SESSION, 0); // we have 666*2 cores.
- LOG.info("Finished testQueryCluster()");
+ public void testQueryCluster() throws Exception {
+ runTest(() -> {
+ LOG.info("Starting testQueryCluster()");
+ runWithArgs(new String[]{"-q"}, "Summary: totalMemory 8192 totalCores 1332", null, RunTypes.YARN_SESSION, 0); // we have 666*2 cores.
+ LOG.info("Finished testQueryCluster()");
+ });
}
/**
@@ -233,18 +240,20 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
*/
@Ignore("The test is too resource consuming (8.5 GB of memory)")
@Test
- public void testResourceComputation() throws IOException {
- addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
- LOG.info("Starting testResourceComputation()");
- runWithArgs(new String[]{
+ public void testResourceComputation() throws Exception {
+ runTest(() -> {
+ addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
+ LOG.info("Starting testResourceComputation()");
+ runWithArgs(new String[]{
"-j", flinkUberjar.getAbsolutePath(),
"-t", flinkLibFolder.getAbsolutePath(),
"-t", flinkShadedHadoopDir.getAbsolutePath(),
"-n", "5",
"-jm", "256m",
"-tm", "1585m"}, "Number of connected TaskManagers changed to", null, RunTypes.YARN_SESSION, 0);
- LOG.info("Finished testResourceComputation()");
- checkForLogString("This YARN session requires 8437MB of memory in the cluster. There are currently only 8192MB available.");
+ LOG.info("Finished testResourceComputation()");
+ checkForLogString("This YARN session requires 8437MB of memory in the cluster. There are currently only 8192MB available.");
+ });
}
/**
@@ -264,18 +273,20 @@ public class YARNSessionFIFOITCase extends YarnTestBase {
*/
@Ignore("The test is too resource consuming (8 GB of memory)")
@Test
- public void testfullAlloc() throws IOException {
- addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
- LOG.info("Starting testfullAlloc()");
- runWithArgs(new String[]{
+ public void testfullAlloc() throws Exception {
+ runTest(() -> {
+ addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN);
+ LOG.info("Starting testfullAlloc()");
+ runWithArgs(new String[]{
"-j", flinkUberjar.getAbsolutePath(),
"-t", flinkLibFolder.getAbsolutePath(),
"-t", flinkShadedHadoopDir.getAbsolutePath(),
"-n", "2",
"-jm", "256m",
"-tm", "3840m"}, "Number of connected TaskManagers changed to", null, RunTypes.YARN_SESSION, 0);
- LOG.info("Finished testfullAlloc()");
- checkForLogString("There is not enough memory available in the YARN cluster. The TaskManager(s) require 3840MB each. NodeManagers available: [4096, 4096]\n" +
+ LOG.info("Finished testfullAlloc()");
+ checkForLogString("There is not enough memory available in the YARN cluster. The TaskManager(s) require 3840MB each. NodeManagers available: [4096, 4096]\n" +
"After allocating the JobManager (512MB) and (1/2) TaskManagers, the following NodeManagers are available: [3584, 256]");
+ });
}
}
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
index 54a5532..3faf777 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java
@@ -40,7 +40,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
@@ -100,41 +99,43 @@ public class YARNSessionFIFOSecuredITCase extends YARNSessionFIFOITCase {
}
@AfterClass
- public static void teardownSecureCluster() throws Exception {
+ public static void teardownSecureCluster() {
LOG.info("tearing down secure cluster environment");
SecureTestEnvironment.cleanup();
}
@Test(timeout = 60000) // timeout after a minute.
@Override
- public void testDetachedMode() throws InterruptedException, IOException {
- super.testDetachedMode();
- final String[] mustHave = {"Login successful for user", "using keytab file"};
- final boolean jobManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
- mustHave,
- "jobmanager.log");
- final boolean taskManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
- mustHave, "taskmanager.log");
-
- Assert.assertThat(
- "The JobManager and the TaskManager should both run with Kerberos.",
- jobManagerRunsWithKerberos && taskManagerRunsWithKerberos,
- Matchers.is(true));
-
- final List<String> amRMTokens = Lists.newArrayList(AMRMTokenIdentifier.KIND_NAME.toString());
- final String jobmanagerContainerId = getContainerIdByLogName("jobmanager.log");
- final String taskmanagerContainerId = getContainerIdByLogName("taskmanager.log");
- final boolean jobmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, jobmanagerContainerId);
- final boolean taskmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, taskmanagerContainerId);
-
- Assert.assertThat(
- "The JobManager should have AMRMToken.",
- jobmanagerWithAmRmToken,
- Matchers.is(true));
- Assert.assertThat(
- "The TaskManager should not have AMRMToken.",
- taskmanagerWithAmRmToken,
- Matchers.is(false));
+ public void testDetachedMode() throws Exception {
+ runTest(() -> {
+ runDetachedModeTest();
+ final String[] mustHave = {"Login successful for user", "using keytab file"};
+ final boolean jobManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
+ mustHave,
+ "jobmanager.log");
+ final boolean taskManagerRunsWithKerberos = verifyStringsInNamedLogFiles(
+ mustHave, "taskmanager.log");
+
+ Assert.assertThat(
+ "The JobManager and the TaskManager should both run with Kerberos.",
+ jobManagerRunsWithKerberos && taskManagerRunsWithKerberos,
+ Matchers.is(true));
+
+ final List<String> amRMTokens = Lists.newArrayList(AMRMTokenIdentifier.KIND_NAME.toString());
+ final String jobmanagerContainerId = getContainerIdByLogName("jobmanager.log");
+ final String taskmanagerContainerId = getContainerIdByLogName("taskmanager.log");
+ final boolean jobmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, jobmanagerContainerId);
+ final boolean taskmanagerWithAmRmToken = verifyTokenKindInContainerCredentials(amRMTokens, taskmanagerContainerId);
+
+ Assert.assertThat(
+ "The JobManager should have AMRMToken.",
+ jobmanagerWithAmRmToken,
+ Matchers.is(true));
+ Assert.assertThat(
+ "The TaskManager should not have AMRMToken.",
+ taskmanagerWithAmRmToken,
+ Matchers.is(false));
+ });
}
/* For secure cluster testing, it is enough to run only one test and override below test methods
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
index acca620..28653b6 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
@@ -78,124 +78,126 @@ public class YarnConfigurationITCase extends YarnTestBase {
*/
@Test(timeout = 60000)
public void testFlinkContainerMemory() throws Exception {
- final YarnClient yarnClient = getYarnClient();
- final Configuration configuration = new Configuration(flinkConfiguration);
+ runTest(() -> {
+ final YarnClient yarnClient = getYarnClient();
+ final Configuration configuration = new Configuration(flinkConfiguration);
- final int masterMemory = 64;
- final int taskManagerMemory = 128;
- final int slotsPerTaskManager = 3;
+ final int masterMemory = 64;
+ final int taskManagerMemory = 128;
+ final int slotsPerTaskManager = 3;
- // disable heap cutoff min
- configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);
- configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(1L << 20));
- configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(4L << 20));
+ // disable heap cutoff min
+ configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);
+ configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(1L << 20));
+ configuration.setString(NetworkEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(4L << 20));
- final YarnConfiguration yarnConfiguration = getYarnConfiguration();
- final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
- configuration,
- yarnConfiguration,
- CliFrontend.getConfigurationDirectoryFromEnv(),
- yarnClient,
- true);
+ final YarnConfiguration yarnConfiguration = getYarnConfiguration();
+ final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
+ configuration,
+ yarnConfiguration,
+ CliFrontend.getConfigurationDirectoryFromEnv(),
+ yarnClient,
+ true);
- clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
- clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
- clusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
+ clusterDescriptor.setLocalJarPath(new Path(flinkUberjar.getAbsolutePath()));
+ clusterDescriptor.addShipFiles(Arrays.asList(flinkLibFolder.listFiles()));
+ clusterDescriptor.addShipFiles(Arrays.asList(flinkShadedHadoopDir.listFiles()));
- final File streamingWordCountFile = getTestJarPath("WindowJoin.jar");
+ final File streamingWordCountFile = getTestJarPath("WindowJoin.jar");
- final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile);
- final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1);
+ final PackagedProgram packagedProgram = new PackagedProgram(streamingWordCountFile);
+ final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(packagedProgram, configuration, 1);
- try {
- final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
- .setMasterMemoryMB(masterMemory)
- .setTaskManagerMemoryMB(taskManagerMemory)
- .setSlotsPerTaskManager(slotsPerTaskManager)
- .createClusterSpecification();
+ try {
+ final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
+ .setMasterMemoryMB(masterMemory)
+ .setTaskManagerMemoryMB(taskManagerMemory)
+ .setSlotsPerTaskManager(slotsPerTaskManager)
+ .createClusterSpecification();
- final ClusterClient<ApplicationId> clusterClient = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true);
+ final ClusterClient<ApplicationId> clusterClient = clusterDescriptor.deployJobCluster(clusterSpecification, jobGraph, true);
- final ApplicationId clusterId = clusterClient.getClusterId();
+ final ApplicationId clusterId = clusterClient.getClusterId();
- final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration), TestingUtils.defaultExecutor());
+ final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(configuration), TestingUtils.defaultExecutor());
- try {
- final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId);
+ try {
+ final ApplicationReport applicationReport = yarnClient.getApplicationReport(clusterId);
- final ApplicationAttemptId currentApplicationAttemptId = applicationReport.getCurrentApplicationAttemptId();
+ final ApplicationAttemptId currentApplicationAttemptId = applicationReport.getCurrentApplicationAttemptId();
- // wait until we have second container allocated
- List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId);
+ // wait until we have second container allocated
+ List<ContainerReport> containers = yarnClient.getContainers(currentApplicationAttemptId);
- while (containers.size() < 2) {
- // this is nasty but Yarn does not offer a better way to wait
- Thread.sleep(50L);
- containers = yarnClient.getContainers(currentApplicationAttemptId);
- }
+ while (containers.size() < 2) {
+ // this is nasty but Yarn does not offer a better way to wait
+ Thread.sleep(50L);
+ containers = yarnClient.getContainers(currentApplicationAttemptId);
+ }
- for (ContainerReport container : containers) {
- if (container.getContainerId().getId() == 1) {
- // this should be the application master
- assertThat(container.getAllocatedResource().getMemory(), is(masterMemory));
- } else {
- assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory));
+ for (ContainerReport container : containers) {
+ if (container.getContainerId().getId() == 1) {
+ // this should be the application master
+ assertThat(container.getAllocatedResource().getMemory(), is(masterMemory));
+ } else {
+ assertThat(container.getAllocatedResource().getMemory(), is(taskManagerMemory));
+ }
}
- }
- final URI webURI = new URI(clusterClient.getWebInterfaceURL());
+ final URI webURI = new URI(clusterClient.getWebInterfaceURL());
- CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture;
- Collection<TaskManagerInfo> taskManagerInfos;
+ CompletableFuture<TaskManagersInfo> taskManagersInfoCompletableFuture;
+ Collection<TaskManagerInfo> taskManagerInfos;
- while (true) {
- taskManagersInfoCompletableFuture = restClient.sendRequest(
- webURI.getHost(),
- webURI.getPort(),
- TaskManagersHeaders.getInstance(),
- EmptyMessageParameters.getInstance(),
- EmptyRequestBody.getInstance());
+ while (true) {
+ taskManagersInfoCompletableFuture = restClient.sendRequest(
+ webURI.getHost(),
+ webURI.getPort(),
+ TaskManagersHeaders.getInstance(),
+ EmptyMessageParameters.getInstance(),
+ EmptyRequestBody.getInstance());
- final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get();
+ final TaskManagersInfo taskManagersInfo = taskManagersInfoCompletableFuture.get();
- taskManagerInfos = taskManagersInfo.getTaskManagerInfos();
+ taskManagerInfos = taskManagersInfo.getTaskManagerInfos();
- // wait until the task manager has registered and reported its slots
- if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) {
- break;
- } else {
- Thread.sleep(100L);
+ // wait until the task manager has registered and reported its slots
+ if (hasTaskManagerConnectedAndReportedSlots(taskManagerInfos)) {
+ break;
+ } else {
+ Thread.sleep(100L);
+ }
}
- }
- // there should be at least one TaskManagerInfo
- final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
+ // there should be at least one TaskManagerInfo
+ final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
- assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager));
+ assertThat(taskManagerInfo.getNumberSlots(), is(slotsPerTaskManager));
- final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters.create(
- configuration,
- taskManagerMemory,
- slotsPerTaskManager);
+ final ContaineredTaskManagerParameters containeredTaskManagerParameters = ContaineredTaskManagerParameters.create(
+ configuration,
+ taskManagerMemory,
+ slotsPerTaskManager);
- final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L;
+ final long expectedHeadSize = containeredTaskManagerParameters.taskManagerHeapSizeMB() << 20L;
- // We compare here physical memory assigned to a container with the heap memory that we should pass to
- // jvm as Xmx parameter. Those value might differ significantly due to system page size or jvm
- // implementation therefore we use 15% threshold here.
- assertThat(
- (double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize,
- is(closeTo(1.0, 0.15)));
- } finally {
- restClient.shutdown(TIMEOUT);
- clusterClient.shutdown();
- }
+ // We compare here physical memory assigned to a container with the heap memory that we should pass to
+ // jvm as Xmx parameter. Those value might differ significantly due to system page size or jvm
+ // implementation therefore we use 15% threshold here.
+ assertThat(
+ (double) taskManagerInfo.getHardwareDescription().getSizeOfJvmHeap() / (double) expectedHeadSize,
+ is(closeTo(1.0, 0.15)));
+ } finally {
+ restClient.shutdown(TIMEOUT);
+ clusterClient.shutdown();
+ }
- clusterDescriptor.killCluster(clusterId);
+ clusterDescriptor.killCluster(clusterId);
- } finally {
- clusterDescriptor.close();
- }
+ } finally {
+ clusterDescriptor.close();
+ }
+ });
}
private boolean hasTaskManagerConnectedAndReportedSlots(Collection<TaskManagerInfo> taskManagerInfos) {
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
index 9beaf49..42e3b43 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.runtime.clusterframework.BootstrapTools;
import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.TestLogger;
+import org.apache.flink.util.function.RunnableWithException;
import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
import org.apache.commons.io.FileUtils;
@@ -42,7 +43,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -200,7 +200,7 @@ public abstract class YarnTestBase extends TestLogger {
}
@Before
- public void checkClusterEmpty() {
+ public void setupYarnClient() {
if (yarnClient == null) {
yarnClient = YarnClient.createYarnClient();
yarnClient.init(getYarnConfiguration());
@@ -214,29 +214,43 @@ public abstract class YarnTestBase extends TestLogger {
* Sleep a bit between the tests (we are re-using the YARN cluster for the tests).
*/
@After
- public void sleep() throws IOException, YarnException {
- Deadline deadline = Deadline.now().plus(Duration.ofSeconds(10));
+ public void shutdownYarnClient() {
+ yarnClient.stop();
+ }
- boolean isAnyJobRunning = yarnClient.getApplications().stream()
- .anyMatch(YarnTestBase::isApplicationRunning);
+ protected void runTest(RunnableWithException test) throws Exception {
+ // wrapping the cleanup logic in an AutoClosable automatically suppresses additional exceptions
+ try (final CleanupYarnApplication ignored = new CleanupYarnApplication()) {
+ test.run();
+ }
+ }
- while (deadline.hasTimeLeft() && isAnyJobRunning) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- Assert.fail("Should not happen");
- }
- isAnyJobRunning = yarnClient.getApplications().stream()
+ private class CleanupYarnApplication implements AutoCloseable {
+ @Override
+ public void close() throws Exception {
+ Deadline deadline = Deadline.now().plus(Duration.ofSeconds(10));
+
+ boolean isAnyJobRunning = yarnClient.getApplications().stream()
.anyMatch(YarnTestBase::isApplicationRunning);
- }
- if (isAnyJobRunning) {
- final List<String> runningApps = yarnClient.getApplications().stream()
- .filter(YarnTestBase::isApplicationRunning)
- .map(app -> "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState() + '.')
- .collect(Collectors.toList());
- if (!runningApps.isEmpty()) {
- Assert.fail("There is at least one application on the cluster that is not finished." + runningApps);
+ while (deadline.hasTimeLeft() && isAnyJobRunning) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ Assert.fail("Should not happen");
+ }
+ isAnyJobRunning = yarnClient.getApplications().stream()
+ .anyMatch(YarnTestBase::isApplicationRunning);
+ }
+
+ if (isAnyJobRunning) {
+ final List<String> runningApps = yarnClient.getApplications().stream()
+ .filter(YarnTestBase::isApplicationRunning)
+ .map(app -> "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState() + '.')
+ .collect(Collectors.toList());
+ if (!runningApps.isEmpty()) {
+ Assert.fail("There is at least one application on the cluster that is not finished." + runningApps);
+ }
}
}
}
[flink] 01/03: [hotfix][yarn][tests] Remove unused field
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 42c8aa56f20a3b358fd974819faeadf75a0dad18
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Fri May 24 10:46:15 2019 +0200
[hotfix][yarn][tests] Remove unused field
---
flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
index 8af5e85..9beaf49 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
@@ -166,8 +166,6 @@ public abstract class YarnTestBase extends TestLogger {
protected org.apache.flink.configuration.Configuration flinkConfiguration;
- protected final boolean isNewMode = true;
-
static {
YARN_CONFIGURATION = new YarnConfiguration();
YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 32);
[flink] 02/03: [FLINK-12445][yarn] Disable test timeout
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e1a4db2841dfd24ff09d6e38fb3a4e988c85a3b8
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Fri May 24 10:46:56 2019 +0200
[FLINK-12445][yarn] Disable test timeout
---
.../java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
index 057ff5a..8e724f4 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
@@ -246,7 +246,7 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
*
* <p><b>Hint: </b> If you think it is a good idea to add more assertions to this test, think again!
*/
- @Test(timeout = 100_000)
+ @Test
public void testVCoresAreSetCorrectlyAndJobManagerHostnameAreShownInWebInterfaceAndDynamicPropertiesAndYarnApplicationNameAndTaskManagerSlots() throws Exception {
checkForProhibitedLogContents = false;
final Runner yarnSessionClusterRunner = startWithArgs(new String[]{