You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2020/05/27 09:04:10 UTC
[flink] branch release-1.11 updated (fe1b533 -> 0e9ddbc)
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a change to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git.
from fe1b533 [FLINK-17934][fs-connector] Add listener to Buckets and remove listener for BucketsBuilder
new 251d577 [hotfix] Correct comment references to flink-yarn-tests
new b54889e [hotfix] Rename org.apache.flink.yarn.util.YarnTestUtils into o.a.f.y.u.TestUtils
new 8b8ed69 [FLINK-17938] Move YarnTestBase.findFile and YarnTestBase.RootDirFilenameFilter to o.a.f.y.u.TestUtils
new 0e9ddbc [FLINK-17750][tests] Harden YARNHighavailabilityITCase by increasing ZK session timeout to 20s
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
flink-yarn-tests/pom.xml | 2 +-
.../flink/yarn/CliFrontendRunWithYarnTest.java | 2 +-
.../test/java/org/apache/flink/yarn/UtilsTest.java | 3 +-
.../apache/flink/yarn/YARNApplicationITCase.java | 2 +-
.../flink/yarn/YARNFileReplicationITCase.java | 4 +-
.../flink/yarn/YARNHighAvailabilityITCase.java | 6 +--
.../java/org/apache/flink/yarn/YARNITCase.java | 4 +-
.../yarn/YARNSessionCapacitySchedulerITCase.java | 5 ++-
.../apache/flink/yarn/YARNSessionFIFOITCase.java | 2 +-
.../apache/flink/yarn/YarnConfigurationITCase.java | 2 +-
.../java/org/apache/flink/yarn/YarnTestBase.java | 50 ++++------------------
.../util/{YarnTestUtils.java => TestUtils.java} | 35 ++++++++++++++-
12 files changed, 60 insertions(+), 57 deletions(-)
rename flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/{YarnTestUtils.java => TestUtils.java} (68%)
[flink] 01/04: [hotfix] Correct comment references to
flink-yarn-tests
Posted by tr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 251d577bc39dc3b6c98d2a76e863b3b9c2af9709
Author: Till Rohrmann <tr...@apache.org>
AuthorDate: Tue May 26 14:15:19 2020 +0200
[hotfix] Correct comment references to flink-yarn-tests
---
flink-yarn-tests/pom.xml | 2 +-
flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index a72a988..617e4fc 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -411,7 +411,7 @@ under the License.
<overWriteSnapshots>true</overWriteSnapshots>
</configuration>
</execution>
- <!-- Write classpath of flink-yarn to a file, so that the yarn tests can use it as their classpath
+ <!-- Write classpath of flink-yarn-tests to a file, so that the yarn tests can use it as their classpath
for the YARN "containers".
-->
<execution>
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
index 031e139..fc52aa1 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
@@ -201,7 +201,7 @@ public abstract class YarnTestBase extends TestLogger {
/**
* Searches for the yarn.classpath file generated by the "dependency:build-classpath" maven plugin in
- * "flink-yarn".
+ * "flink-yarn-tests".
* @return a classpath suitable for running all YARN-launched JVMs
*/
private static String getYarnClasspath() {
[flink] 04/04: [FLINK-17750][tests] Harden
YARNHighavailabilityITCase by increasing ZK session timeout to 20s
Posted by tr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 0e9ddbcbfae556860c0c5323e616cc82f55efe6a
Author: Till Rohrmann <tr...@apache.org>
AuthorDate: Tue May 26 16:21:35 2020 +0200
[FLINK-17750][tests] Harden YARNHighavailabilityITCase by increasing ZK session timeout to 20s
In order to harden YARNHighAvailabilityITCase against ZK session timeouts, the timeout has been increased
from 1s to 20s. This increase also increases the runtime of YARNHighAvailabilityITCase.testKillYarnSessionClusterEntrypoint
slightly.
This closes #12347.
---
.../src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index eba7140..6935ce3 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -274,7 +274,7 @@ public class YARNHighAvailabilityITCase extends YarnTestBase {
flinkConfiguration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
flinkConfiguration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, storageDir);
flinkConfiguration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
- flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 1000);
+ flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 20000);
flinkConfiguration.setString(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay");
flinkConfiguration.setInteger(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, Integer.MAX_VALUE);
[flink] 02/04: [hotfix] Rename
org.apache.flink.yarn.util.YarnTestUtils into o.a.f.y.u.TestUtils
Posted by tr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git
commit b54889edd2e59879ae9527dec1cd682475640bee
Author: Till Rohrmann <tr...@apache.org>
AuthorDate: Tue May 26 14:22:48 2020 +0200
[hotfix] Rename org.apache.flink.yarn.util.YarnTestUtils into o.a.f.y.u.TestUtils
Renamed o.a.f.y.u.YarnTestUtils into o.a.f.y.u.TestUtils in order to avoid a naming conflict with
o.a.f.y.YarnTestUtils from the flink-yarn module.
---
.../test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java | 2 +-
.../src/test/java/org/apache/flink/yarn/YARNApplicationITCase.java | 2 +-
.../test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java | 4 ++--
.../test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java | 4 ++--
flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java | 4 ++--
.../org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java | 2 +-
.../src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java | 2 +-
.../src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java | 2 +-
.../org/apache/flink/yarn/util/{YarnTestUtils.java => TestUtils.java} | 2 +-
9 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java
index 7e14ac7..82ead7e 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java
@@ -36,7 +36,7 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import static org.apache.flink.client.cli.CliFrontendRunTest.verifyCliFrontend;
-import static org.apache.flink.yarn.util.YarnTestUtils.getTestJarPath;
+import static org.apache.flink.yarn.util.TestUtils.getTestJarPath;
/**
* Tests for the RUN command using a {@link FlinkYarnSessionCli} inside the {@link CliFrontend}.
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNApplicationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNApplicationITCase.java
index 13e59c5..5aa5203 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNApplicationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNApplicationITCase.java
@@ -41,7 +41,7 @@ import java.time.Duration;
import java.util.Collections;
import static org.apache.flink.yarn.configuration.YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR;
-import static org.apache.flink.yarn.util.YarnTestUtils.getTestJarPath;
+import static org.apache.flink.yarn.util.TestUtils.getTestJarPath;
/**
* Test cases for the deployment of Yarn Flink application clusters.
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java
index b2d6da5..3b0ea5b 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java
@@ -30,7 +30,7 @@ import org.apache.flink.runtime.jobmaster.JobResult;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
-import org.apache.flink.yarn.util.YarnTestUtils;
+import org.apache.flink.yarn.util.TestUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -91,7 +91,7 @@ public class YARNFileReplicationITCase extends YarnTestBase {
.setSlotsPerTaskManager(1)
.createClusterSpecification();
- File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));
+ File testingJar = YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index b3a9269..934df92 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -43,7 +43,7 @@ import org.apache.flink.util.OperatingSystem;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
import org.apache.flink.yarn.entrypoint.YarnSessionClusterEntrypoint;
import org.apache.flink.yarn.testjob.YarnTestJob;
-import org.apache.flink.yarn.util.YarnTestUtils;
+import org.apache.flink.yarn.util.TestUtils;
import org.apache.flink.shaded.guava18.com.google.common.collect.Iterables;
@@ -140,7 +140,7 @@ public class YARNHighAvailabilityITCase extends YarnTestBase {
stopJobSignal = YarnTestJob.StopJobSignal.usingMarkerFile(FOLDER.newFile().toPath());
job = YarnTestJob.stoppableJob(stopJobSignal);
final File testingJar =
- YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));
+ YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
assertThat(testingJar, notNullValue());
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
index aa1f0a7..47e523c 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
@@ -31,7 +31,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
import org.apache.flink.yarn.testjob.YarnTestCacheJob;
-import org.apache.flink.yarn.util.YarnTestUtils;
+import org.apache.flink.yarn.util.TestUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -116,7 +116,7 @@ public class YARNITCase extends YarnTestBase {
.setSlotsPerTaskManager(1)
.createClusterSpecification();
- File testingJar = YarnTestBase.findFile("..", new YarnTestUtils.TestJarFinder("flink-yarn-tests"));
+ File testingJar = YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
index fe1e62a..4145425 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
@@ -84,7 +84,7 @@ import java.util.stream.Collectors;
import static junit.framework.TestCase.assertTrue;
import static org.apache.flink.util.Preconditions.checkState;
-import static org.apache.flink.yarn.util.YarnTestUtils.getTestJarPath;
+import static org.apache.flink.yarn.util.TestUtils.getTestJarPath;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasItem;
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
index bb1d98b..fafc06e 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java
@@ -51,7 +51,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import static org.apache.flink.yarn.util.YarnTestUtils.getTestJarPath;
+import static org.apache.flink.yarn.util.TestUtils.getTestJarPath;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.hasItem;
import static org.junit.Assert.assertThat;
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
index cceb3a0..19f3d52 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java
@@ -58,7 +58,7 @@ import java.util.Collection;
import java.util.List;
import java.util.concurrent.CompletableFuture;
-import static org.apache.flink.yarn.util.YarnTestUtils.getTestJarPath;
+import static org.apache.flink.yarn.util.TestUtils.getTestJarPath;
import static org.hamcrest.Matchers.closeTo;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/YarnTestUtils.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java
similarity index 98%
rename from flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/YarnTestUtils.java
rename to flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java
index f7a9634..5d852de 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/YarnTestUtils.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java
@@ -25,7 +25,7 @@ import java.io.FilenameFilter;
/**
* Utility methods for YARN tests.
*/
-public class YarnTestUtils {
+public class TestUtils {
public static File getTestJarPath(String fileName) throws FileNotFoundException {
File f = new File("target/programs/" + fileName);
if (!f.exists()) {
[flink] 03/04: [FLINK-17938] Move YarnTestBase.findFile and
YarnTestBase.RootDirFilenameFilter to o.a.f.y.u.TestUtils
Posted by tr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch release-1.11
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 8b8ed695714d878fe69ca3dad8e7540b41c5d4e7
Author: Till Rohrmann <tr...@apache.org>
AuthorDate: Tue May 26 14:28:09 2020 +0200
[FLINK-17938] Move YarnTestBase.findFile and YarnTestBase.RootDirFilenameFilter to o.a.f.y.u.TestUtils
Moving findFile and RootDirFilenameFilter to o.a.f.y.u.TestUtils allows to run the UtilsTest suite without
requiring that yarn.classpath has been generated. The latter is required whenever one uses the YarnTestBase
class because it reads this file when the class is being loaded.
This closes #12341.
---
.../test/java/org/apache/flink/yarn/UtilsTest.java | 3 +-
.../flink/yarn/YARNFileReplicationITCase.java | 2 +-
.../flink/yarn/YARNHighAvailabilityITCase.java | 2 +-
.../java/org/apache/flink/yarn/YARNITCase.java | 2 +-
.../yarn/YARNSessionCapacitySchedulerITCase.java | 3 +-
.../java/org/apache/flink/yarn/YarnTestBase.java | 48 ++++------------------
.../java/org/apache/flink/yarn/util/TestUtils.java | 33 +++++++++++++++
7 files changed, 48 insertions(+), 45 deletions(-)
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java
index 53fd6ba..76aac0e 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameter
import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec;
import org.apache.flink.runtime.clusterframework.TaskExecutorProcessUtils;
import org.apache.flink.util.TestLogger;
+import org.apache.flink.yarn.util.TestUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -66,7 +67,7 @@ public class UtilsTest extends TestLogger {
@Test
public void testUberjarLocator() {
- File dir = YarnTestBase.findFile("..", new YarnTestBase.RootDirFilenameFilter());
+ File dir = TestUtils.findFile("..", new TestUtils.RootDirFilenameFilter());
Assert.assertNotNull(dir);
Assert.assertTrue(dir.getName().endsWith(".jar"));
dir = dir.getParentFile().getParentFile(); // from uberjar to lib to root
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java
index 3b0ea5b..b1f4560 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java
@@ -91,7 +91,7 @@ public class YARNFileReplicationITCase extends YarnTestBase {
.setSlotsPerTaskManager(1)
.createClusterSpecification();
- File testingJar = YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
+ File testingJar = TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
index 934df92..eba7140 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java
@@ -140,7 +140,7 @@ public class YARNHighAvailabilityITCase extends YarnTestBase {
stopJobSignal = YarnTestJob.StopJobSignal.usingMarkerFile(FOLDER.newFile().toPath());
job = YarnTestJob.stoppableJob(stopJobSignal);
final File testingJar =
- YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
+ TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
assertThat(testingJar, notNullValue());
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
index 47e523c..1e5a64e 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java
@@ -116,7 +116,7 @@ public class YARNITCase extends YarnTestBase {
.setSlotsPerTaskManager(1)
.createClusterSpecification();
- File testingJar = YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
+ File testingJar = TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests"));
jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI()));
try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
index 4145425..af0f532 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java
@@ -40,6 +40,7 @@ import org.apache.flink.testutils.logging.TestLoggerResource;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
import org.apache.flink.yarn.configuration.YarnConfigOptions;
+import org.apache.flink.yarn.util.TestUtils;
import org.apache.flink.shaded.guava18.com.google.common.net.HostAndPort;
@@ -561,7 +562,7 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase {
Assert.assertTrue("Expected string 'der 29' or '(mind,1)' not found in string'" + content + "'", content.contains("der 29") || content.contains("(der,29)") || content.contains("(mind,1)"));
// check if the heap size for the TaskManager was set correctly
- File jobmanagerLog = YarnTestBase.findFile("..", new FilenameFilter() {
+ File jobmanagerLog = TestUtils.findFile("..", new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.contains("jobmanager.log") && dir.getAbsolutePath().contains(id.toString());
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
index fc52aa1..dde1fe2 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java
@@ -28,6 +28,7 @@ import org.apache.flink.util.Preconditions;
import org.apache.flink.util.TestLogger;
import org.apache.flink.util.function.RunnableWithException;
import org.apache.flink.yarn.cli.FlinkYarnSessionCli;
+import org.apache.flink.yarn.util.TestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@@ -207,7 +208,7 @@ public abstract class YarnTestBase extends TestLogger {
private static String getYarnClasspath() {
final String start = "../flink-yarn-tests";
try {
- File classPathFile = findFile(start, (dir, name) -> name.equals("yarn.classpath"));
+ File classPathFile = TestUtils.findFile(start, (dir, name) -> name.equals("yarn.classpath"));
return FileUtils.readFileToString(classPathFile); // potential NPE is supposed to be fatal
} catch (Throwable t) {
LOG.error("Error while getting YARN classpath in {}", new File(start).getAbsoluteFile(), t);
@@ -305,29 +306,6 @@ public abstract class YarnTestBase extends TestLogger {
return YARN_CONFIGURATION;
}
- /**
- * Locate a file or directory.
- */
- public static File findFile(String startAt, FilenameFilter fnf) {
- File root = new File(startAt);
- String[] files = root.list();
- if (files == null) {
- return null;
- }
- for (String file : files) {
- File f = new File(startAt + File.separator + file);
- if (f.isDirectory()) {
- File r = findFile(f.getAbsolutePath(), fnf);
- if (r != null) {
- return r;
- }
- } else if (fnf.accept(f.getParentFile(), f.getName())) {
- return f;
- }
- }
- return null;
- }
-
@Nonnull
YarnClusterDescriptor createYarnClusterDescriptor(org.apache.flink.configuration.Configuration flinkConfiguration) {
final YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptorWithoutLibDir(flinkConfiguration);
@@ -347,16 +325,6 @@ public abstract class YarnTestBase extends TestLogger {
}
/**
- * Filter to find root dir of the flink-yarn dist.
- */
- public static class RootDirFilenameFilter implements FilenameFilter {
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith("flink-dist") && name.endsWith(".jar") && dir.toString().contains("/lib");
- }
- }
-
- /**
* A simple {@link FilenameFilter} that only accepts files if their name contains every string in the array passed
* to the constructor.
*/
@@ -428,7 +396,7 @@ public abstract class YarnTestBase extends TestLogger {
Assert.assertTrue("Expecting directory " + cwd.getAbsolutePath() + " to be a directory", cwd.isDirectory());
List<String> prohibitedExcerpts = new ArrayList<>();
- File foundFile = findFile(cwd.getAbsolutePath(), new FilenameFilter() {
+ File foundFile = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
// scan each file for prohibited strings.
@@ -519,7 +487,7 @@ public abstract class YarnTestBase extends TestLogger {
return false;
}
- File foundFile = findFile(cwd.getAbsolutePath(), new FilenameFilter() {
+ File foundFile = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
if (fileName != null && !name.equals(fileName)) {
@@ -562,7 +530,7 @@ public abstract class YarnTestBase extends TestLogger {
return false;
}
- File containerTokens = findFile(cwd.getAbsolutePath(), new FilenameFilter() {
+ File containerTokens = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.equals(containerId + ".tokens");
@@ -589,7 +557,7 @@ public abstract class YarnTestBase extends TestLogger {
public static String getContainerIdByLogName(String logName) {
File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY));
- File containerLog = findFile(cwd.getAbsolutePath(), new FilenameFilter() {
+ File containerLog = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.equals(logName);
@@ -653,7 +621,7 @@ public abstract class YarnTestBase extends TestLogger {
System.setProperty("user.home", homeDir.getAbsolutePath());
String uberjarStartLoc = "..";
LOG.info("Trying to locate uberjar in {}", new File(uberjarStartLoc).getAbsolutePath());
- flinkUberjar = findFile(uberjarStartLoc, new RootDirFilenameFilter());
+ flinkUberjar = TestUtils.findFile(uberjarStartLoc, new TestUtils.RootDirFilenameFilter());
Assert.assertNotNull("Flink uberjar not found", flinkUberjar);
String flinkDistRootDir = flinkUberjar.getParentFile().getParent();
flinkLibFolder = flinkUberjar.getParentFile(); // the uberjar is located in lib/
@@ -681,7 +649,7 @@ public abstract class YarnTestBase extends TestLogger {
Map<String, String> map = new HashMap<String, String>(System.getenv());
- File flinkConfDirPath = findFile(flinkDistRootDir, new ContainsName(new String[]{"flink-conf.yaml"}));
+ File flinkConfDirPath = TestUtils.findFile(flinkDistRootDir, new ContainsName(new String[]{"flink-conf.yaml"}));
Assert.assertNotNull(flinkConfDirPath);
final String confDirPath = flinkConfDirPath.getParentFile().getAbsolutePath();
diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java
index 5d852de..a8ae69a 100644
--- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java
+++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java
@@ -36,6 +36,29 @@ public class TestUtils {
}
/**
+ * Locate a file or directory.
+ */
+ public static File findFile(String startAt, FilenameFilter fnf) {
+ File root = new File(startAt);
+ String[] files = root.list();
+ if (files == null) {
+ return null;
+ }
+ for (String file : files) {
+ File f = new File(startAt + File.separator + file);
+ if (f.isDirectory()) {
+ File r = findFile(f.getAbsolutePath(), fnf);
+ if (r != null) {
+ return r;
+ }
+ } else if (fnf.accept(f.getParentFile(), f.getName())) {
+ return f;
+ }
+ }
+ return null;
+ }
+
+ /**
* Filename filter which finds the test jar for the given name.
*/
public static class TestJarFinder implements FilenameFilter {
@@ -52,4 +75,14 @@ public class TestUtils {
dir.getAbsolutePath().contains(File.separator + jarName + File.separator);
}
}
+
+ /**
+ * Filter to find root dir of the flink-yarn dist.
+ */
+ public static class RootDirFilenameFilter implements FilenameFilter {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.startsWith("flink-dist") && name.endsWith(".jar") && dir.toString().contains("/lib");
+ }
+ }
}