You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/05/29 11:19:16 UTC

[5/6] flink git commit: [FLINK-9383][runtime] Test directories in DistributedCache E2E test

[FLINK-9383][runtime] Test directories in DistributedCache E2E test


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c0cdc42b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c0cdc42b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c0cdc42b

Branch: refs/heads/master
Commit: c0cdc42bc72c88d96fb8281a55ac0e8b8759093f
Parents: 2a7c336
Author: zentol <ch...@apache.org>
Authored: Tue May 15 13:57:56 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Tue May 29 09:33:50 2018 +0200

----------------------------------------------------------------------
 .../DistributedCacheViaBlobTestProgram.java     | 37 ++++++++++++++++++--
 ...test_streaming_distributed_cache_via_blob.sh |  2 +-
 2 files changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/c0cdc42b/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java b/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
index 9ad4b98..167101c 100644
--- a/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
+++ b/flink-end-to-end-tests/flink-distributed-cache-via-blob-test/src/main/java/org/apache/flink/streaming/tests/DistributedCacheViaBlobTestProgram.java
@@ -39,14 +39,23 @@ public class DistributedCacheViaBlobTestProgram {
 		final ParameterTool params = ParameterTool.fromArgs(args);
 
 		final Path inputFile = Paths.get(params.getRequired("inputFile"));
+		final Path inputDir = Paths.get(params.getRequired("inputDir"));
 
 		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(1);
 
 		env.registerCachedFile(inputFile.toString(), "test_data", false);
+		env.registerCachedFile(inputDir.toString(), "test_dir", false);
+
+		Path containedFile = Files.list(inputDir).findAny()
+			.orElseThrow(() -> new RuntimeException("Input directory must not be empty."));
 
 		env.fromElements(1)
-			.map(new TestMapFunction(inputFile.toAbsolutePath().toString()))
+			.map(new TestMapFunction(
+				inputFile.toAbsolutePath().toString(),
+				Files.size(inputFile),
+				inputDir.toAbsolutePath().toString(),
+				containedFile.getFileName().toString()))
 			.writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
 
 		env.execute("Distributed Cache Via Blob Test Program");
@@ -54,21 +63,43 @@ public class DistributedCacheViaBlobTestProgram {
 
 	static class TestMapFunction extends RichMapFunction<Integer, String> {
 
-		private String initialPath;
+		private final String initialPath;
+		private final long fileSize;
+
+		private final String initialDirPath;
+		private final String containedFileName;
 
-		public TestMapFunction(String initialPath) {
+		public TestMapFunction(String initialPath, long fileSize, String initialDirPath, String containedFileName) {
 			this.initialPath = initialPath;
+			this.fileSize = fileSize;
+			this.initialDirPath = initialDirPath;
+			this.containedFileName = containedFileName;
 		}
 
 		@Override
 		public String map(Integer value) throws Exception {
 			final Path testFile = getRuntimeContext().getDistributedCache().getFile("test_data").toPath();
+			final Path testDir = getRuntimeContext().getDistributedCache().getFile("test_dir").toPath();
 
 			if (testFile.toAbsolutePath().toString().equals(initialPath)) {
 				throw new RuntimeException(String.format("Operator should access copy from cache rather than the " +
 					"initial file. Input file path: %s. Cache file path: %s", initialPath, testFile));
 			}
 
+			long testFileSize = Files.size(testFile);
+			if (testFileSize != fileSize) {
+				throw new RuntimeException(String.format("File size does not match. Expected:%s Actual:%s", fileSize, testFileSize));
+			}
+
+			if (testDir.toAbsolutePath().toString().equals(initialDirPath)) {
+				throw new RuntimeException(String.format("Operator should access copy from cache rather than the " +
+					"initial dir. Input dir path: %s. Cache dir path: %s", initialDirPath, testDir));
+			}
+
+			if (Files.list(testDir).map(Path::getFileName).map(Path::toString).noneMatch(path -> path.equals(containedFileName))) {
+				throw new RuntimeException(String.format("Cached directory %s should not be empty.", testDir));
+			}
+
 			return Files.readAllLines(testFile)
 				.stream()
 				.collect(Collectors.joining("\n"));

http://git-wip-us.apache.org/repos/asf/flink/blob/c0cdc42b/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh b/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
index b85f610..25ae7a0 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_distributed_cache_via_blob.sh
@@ -27,7 +27,7 @@ start_cluster
 
 mkdir -p $TEST_DATA_DIR
 
-$FLINK_DIR/bin/flink run -p 1 $TEST_PROGRAM_JAR --inputFile $TEST_INFRA_DIR/test-data/words --tempDir $TEST_DATA_DIR/ --output $TEST_DATA_DIR/out/cl_out_pf
+$FLINK_DIR/bin/flink run -p 1 $TEST_PROGRAM_JAR --inputFile $TEST_INFRA_DIR/test-data/words --inputDir $TEST_INFRA_DIR/test-data --tempDir $TEST_DATA_DIR/ --output $TEST_DATA_DIR/out/cl_out_pf
 
 OUTPUT=`cat $TEST_DATA_DIR/out/cl_out_pf`