You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/03/10 13:42:45 UTC

[3/7] flink git commit: [FLINK-5976] [tests] Deduplicate Tokenizer in tests

[FLINK-5976] [tests] Deduplicate Tokenizer in tests

This closes #3485


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e5e6da0b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e5e6da0b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e5e6da0b

Branch: refs/heads/master
Commit: e5e6da0b38495d4030fd0b790a81ca70f9ce84b2
Parents: 6f47e31
Author: liuyuzhong7 <li...@gmail.com>
Authored: Tue Mar 7 19:15:40 2017 +0800
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 10 13:30:57 2017 +0100

----------------------------------------------------------------------
 .../clients/examples/LocalExecutorITCase.java   | 29 ++++------------
 .../hadoop/mapred/WordCountMapredITCase.java    | 19 +----------
 .../mapreduce/WordCountMapreduceITCase.java     | 19 +----------
 .../api/outputformat/CsvOutputFormatITCase.java | 21 +-----------
 .../outputformat/TextOutputFormatITCase.java    |  3 +-
 .../flink/test/testfunctions/Tokenizer.java     | 36 ++++++++++++++++++++
 6 files changed, 48 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
index 4ed28a8..95d1ab2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
@@ -20,15 +20,12 @@ package org.apache.flink.test.clients.examples;
 
 import java.io.File;
 import java.io.FileWriter;
-
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.client.LocalExecutor;
 import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -45,7 +42,7 @@ public class LocalExecutorITCase {
 			File outFile = File.createTempFile("wctext", ".out");
 			inFile.deleteOnExit();
 			outFile.deleteOnExit();
-			
+
 			try (FileWriter fw = new FileWriter(inFile)) {
 				fw.write(WordCountData.TEXT);
 			}
@@ -64,27 +61,15 @@ public class LocalExecutorITCase {
 			Assert.fail(e.getMessage());
 		}
 	}
-	
+
 	private Plan getWordCountPlan(File inFile, File outFile, int parallelism) {
 		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 		env.setParallelism(parallelism);
 		env.readTextFile(inFile.getAbsolutePath())
-				.flatMap(new Tokenizer())
-				.groupBy(0)
-				.sum(1)
-				.writeAsCsv(outFile.getAbsolutePath());
+			.flatMap(new Tokenizer())
+			.groupBy(0)
+			.sum(1)
+			.writeAsCsv(outFile.getAbsolutePath());
 		return env.createProgramPlan();
 	}
-
-	public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
-		@Override
-		public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
-			String[] tokens = value.toLowerCase().split("\\W+");
-			for (String token : tokens) {
-				if (token.length() > 0) {
-					out.collect(new Tuple2<>(token, 1));
-				}
-			}
-		}
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
index 9528d94..7c1b30e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.test.hadoop.mapred;
 
-import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
@@ -27,7 +26,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
 import static org.apache.flink.hadoopcompatibility.HadoopInputs.readHadoopFile;
 import org.apache.flink.test.testdata.WordCountData;
 import org.apache.flink.test.util.JavaProgramTestBase;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
 import org.apache.flink.util.OperatingSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
@@ -115,20 +114,4 @@ public class WordCountMapredITCase extends JavaProgramTestBase {
 		words.output(hadoopOutputFormat);
 		env.execute("Hadoop Compat WordCount");
 	}
-
-	public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
-		@Override
-		public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
-			// normalize and split the line
-			String[] tokens = value.toLowerCase().split("\\W+");
-
-			// emit the pairs
-			for (String token : tokens) {
-				if (token.length() > 0) {
-					out.collect(new Tuple2<String, Integer>(token, 1));
-				}
-			}
-		}
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
index 64062d2..fbc1994 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.test.hadoop.mapreduce;
 
-import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
@@ -27,7 +26,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
 import static org.apache.flink.hadoopcompatibility.HadoopInputs.readHadoopFile;
 import org.apache.flink.test.testdata.WordCountData;
 import org.apache.flink.test.util.JavaProgramTestBase;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
 import org.apache.flink.util.OperatingSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
@@ -115,20 +114,4 @@ public class WordCountMapreduceITCase extends JavaProgramTestBase {
 		words.output(hadoopOutputFormat);
 		env.execute("Hadoop Compat WordCount");
 	}
-
-	public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
-		@Override
-		public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
-			// normalize and split the line
-			String[] tokens = value.toLowerCase().split("\\W+");
-
-			// emit the pairs
-			for (String token : tokens) {
-				if (token.length() > 0) {
-					out.collect(new Tuple2<String, Integer>(token, 1));
-				}
-			}
-		}
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
index c2155ac..61dddb8 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
@@ -17,13 +17,12 @@
 
 package org.apache.flink.test.streaming.api.outputformat;
 
-import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.util.StreamingProgramTestBase;
 import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
 
 public class CsvOutputFormatITCase extends StreamingProgramTestBase {
 
@@ -56,23 +55,5 @@ public class CsvOutputFormatITCase extends StreamingProgramTestBase {
 				.replaceAll("[\\\\(\\\\)]", ""), resultPath);
 	}
 
-	public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public void flatMap(String value, Collector<Tuple2<String, Integer>> out)
-				throws Exception {
-			// normalize and split the line
-			String[] tokens = value.toLowerCase().split("\\W+");
-
-			// emit the pairs
-			for (String token : tokens) {
-				if (token.length() > 0) {
-					out.collect(new Tuple2<String, Integer>(token, 1));
-				}
-			}
-		}
-	}
-
 }
 

http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
index 2940e6d..6ea864e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
@@ -22,6 +22,7 @@ import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.util.StreamingProgramTestBase;
 import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.test.testfunctions.Tokenizer;
 
 public class TextOutputFormatITCase extends StreamingProgramTestBase {
 
@@ -39,7 +40,7 @@ public class TextOutputFormatITCase extends StreamingProgramTestBase {
 		DataStream<String> text = env.fromElements(WordCountData.TEXT);
 
 		DataStream<Tuple2<String, Integer>> counts = text
-				.flatMap(new CsvOutputFormatITCase.Tokenizer())
+				.flatMap(new Tokenizer())
 				.keyBy(0).sum(1);
 
 		counts.writeAsText(resultPath);

http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java b/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java
new file mode 100644
index 0000000..9b3764d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.testfunctions;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+
+public final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
+
+	@Override
+	public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
+		String[] tokens = value.toLowerCase().split("\\W+");
+		for (String token : tokens) {
+			if (token.length() > 0) {
+				out.collect(new Tuple2<>(token, 1));
+			}
+		}
+	}
+}