You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/05/29 11:19:16 UTC
[5/6] flink git commit: [FLINK-9383][runtime] Test directories in
DistributedCache E2E test
[FLINK-9383][runtime] Test directories in DistributedCache E2E test
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c0cdc42b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c0cdc42b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c0cdc42b
Branch: refs/heads/master
Commit: c0cdc42bc72c88d96fb8281a55ac0e8b8759093f
Parents: 2a7c336
Author: zentol <ch...@apache.org>
Authored: Tue May 15 13:57:56 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Tue May 29 09:33:50 2018 +0200
----------------------------------------------------------------------
.../DistributedCacheViaBlobTestProgram.java | 37 ++++++++++++++++++--
...test_streaming_distributed_cache_via_blob.sh | 2 +-
2 files changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c0cdc42b/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java b/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
index 9ad4b98..167101c 100644
--- a/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
+++ b/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
@@ -39,14 +39,23 @@ public class DistributedCacheViaBlobTestProgram {
final ParameterTool params = ParameterTool.fromArgs(args);
final Path inputFile = Paths.get(params.getRequired("inputFile"));
+ final Path inputDir = Paths.get(params.getRequired("inputDir"));
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(1);
env.registerCachedFile(inputFile.toString(), "test_data", false);
+ env.registerCachedFile(inputDir.toString(), "test_dir", false);
+
+ Path containedFile = Files.list(inputDir).findAny()
+ .orElseThrow(() -> new RuntimeException("Input directory must not be empty."));
env.fromElements(1)
- .map(new TestMapFunction(inputFile.toAbsolutePath().toString()))
+ .map(new TestMapFunction(
+ inputFile.toAbsolutePath().toString(),
+ Files.size(inputFile),
+ inputDir.toAbsolutePath().toString(),
+ containedFile.getFileName().toString()))
.writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
env.execute("Distributed Cache Via Blob Test Program");
@@ -54,21 +63,43 @@ public class DistributedCacheViaBlobTestProgram {
static class TestMapFunction extends RichMapFunction<Integer, String> {
- private String initialPath;
+ private final String initialPath;
+ private final long fileSize;
+
+ private final String initialDirPath;
+ private final String containedFileName;
- public TestMapFunction(String initialPath) {
+ public TestMapFunction(String initialPath, long fileSize, String initialDirPath, String containedFileName) {
this.initialPath = initialPath;
+ this.fileSize = fileSize;
+ this.initialDirPath = initialDirPath;
+ this.containedFileName = containedFileName;
}
@Override
public String map(Integer value) throws Exception {
final Path testFile = getRuntimeContext().getDistributedCache().getFile("test_data").toPath();
+ final Path testDir = getRuntimeContext().getDistributedCache().getFile("test_dir").toPath();
if (testFile.toAbsolutePath().toString().equals(initialPath)) {
throw new RuntimeException(String.format("Operator should access copy from cache rather than the " +
"initial file. Input file path: %s. Cache file path: %s", initialPath, testFile));
}
+ long testFileSize = Files.size(testFile);
+ if (testFileSize != fileSize) {
+ throw new RuntimeException(String.format("File size does not match. Expected:%s Actual:%s", fileSize, testFileSize));
+ }
+
+ if (testDir.toAbsolutePath().toString().equals(initialDirPath)) {
+ throw new RuntimeException(String.format("Operator should access copy from cache rather than the " +
+ "initial dir. Input dir path: %s. Cache dir path: %s", initialDirPath, testDir));
+ }
+
+ if (Files.list(testDir).map(Path::getFileName).map(Path::toString).noneMatch(path -> path.equals(containedFileName))) {
+ throw new RuntimeException(String.format("Cached directory %s should not be empty.", testDir));
+ }
+
return Files.readAllLines(testFile)
.stream()
.collect(Collectors.joining("\n"));
http://git-wip-us.apache.org/repos/asf/flink/blob/c0cdc42b/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh b/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
index b85f610..25ae7a0 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
@@ -27,7 +27,7 @@ start_cluster
mkdir -p $TEST_DATA_DIR
-$FLINK_DIR/bin/flink run -p 1 $TEST_PROGRAM_JAR --inputFile $TEST_INFRA_DIR/test-data/words --tempDir $TEST_DATA_DIR/ --output $TEST_DATA_DIR/out/cl_out_pf
+$FLINK_DIR/bin/flink run -p 1 $TEST_PROGRAM_JAR --inputFile $TEST_INFRA_DIR/test-data/words --inputDir $TEST_INFRA_DIR/test-data --tempDir $TEST_DATA_DIR/ --output $TEST_DATA_DIR/out/cl_out_pf
OUTPUT=`cat $TEST_DATA_DIR/out/cl_out_pf`