You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/03/10 13:42:45 UTC
[3/7] flink git commit: [FLINK-5976] [tests] Deduplicate Tokenizer in
tests
[FLINK-5976] [tests] Deduplicate Tokenizer in tests
This closes #3485
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e5e6da0b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e5e6da0b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e5e6da0b
Branch: refs/heads/master
Commit: e5e6da0b38495d4030fd0b790a81ca70f9ce84b2
Parents: 6f47e31
Author: liuyuzhong7 <li...@gmail.com>
Authored: Tue Mar 7 19:15:40 2017 +0800
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 10 13:30:57 2017 +0100
----------------------------------------------------------------------
.../clients/examples/LocalExecutorITCase.java | 29 ++++------------
.../hadoop/mapred/WordCountMapredITCase.java | 19 +----------
.../mapreduce/WordCountMapreduceITCase.java | 19 +----------
.../api/outputformat/CsvOutputFormatITCase.java | 21 +-----------
.../outputformat/TextOutputFormatITCase.java | 3 +-
.../flink/test/testfunctions/Tokenizer.java | 36 ++++++++++++++++++++
6 files changed, 48 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
index 4ed28a8..95d1ab2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/clients/examples/LocalExecutorITCase.java
@@ -20,15 +20,12 @@ package org.apache.flink.test.clients.examples;
import java.io.File;
import java.io.FileWriter;
-
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.Plan;
-import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.client.LocalExecutor;
import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
import org.junit.Assert;
import org.junit.Test;
@@ -45,7 +42,7 @@ public class LocalExecutorITCase {
File outFile = File.createTempFile("wctext", ".out");
inFile.deleteOnExit();
outFile.deleteOnExit();
-
+
try (FileWriter fw = new FileWriter(inFile)) {
fw.write(WordCountData.TEXT);
}
@@ -64,27 +61,15 @@ public class LocalExecutorITCase {
Assert.fail(e.getMessage());
}
}
-
+
private Plan getWordCountPlan(File inFile, File outFile, int parallelism) {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(parallelism);
env.readTextFile(inFile.getAbsolutePath())
- .flatMap(new Tokenizer())
- .groupBy(0)
- .sum(1)
- .writeAsCsv(outFile.getAbsolutePath());
+ .flatMap(new Tokenizer())
+ .groupBy(0)
+ .sum(1)
+ .writeAsCsv(outFile.getAbsolutePath());
return env.createProgramPlan();
}
-
- public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- String[] tokens = value.toLowerCase().split("\\W+");
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<>(token, 1));
- }
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
index 9528d94..7c1b30e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapred/WordCountMapredITCase.java
@@ -18,7 +18,6 @@
package org.apache.flink.test.hadoop.mapred;
-import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
@@ -27,7 +26,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import static org.apache.flink.hadoopcompatibility.HadoopInputs.readHadoopFile;
import org.apache.flink.test.testdata.WordCountData;
import org.apache.flink.test.util.JavaProgramTestBase;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
import org.apache.flink.util.OperatingSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
@@ -115,20 +114,4 @@ public class WordCountMapredITCase extends JavaProgramTestBase {
words.output(hadoopOutputFormat);
env.execute("Hadoop Compat WordCount");
}
-
- public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
index 64062d2..fbc1994 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/hadoop/mapreduce/WordCountMapreduceITCase.java
@@ -18,7 +18,6 @@
package org.apache.flink.test.hadoop.mapreduce;
-import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
@@ -27,7 +26,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import static org.apache.flink.hadoopcompatibility.HadoopInputs.readHadoopFile;
import org.apache.flink.test.testdata.WordCountData;
import org.apache.flink.test.util.JavaProgramTestBase;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
import org.apache.flink.util.OperatingSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
@@ -115,20 +114,4 @@ public class WordCountMapreduceITCase extends JavaProgramTestBase {
words.output(hadoopOutputFormat);
env.execute("Hadoop Compat WordCount");
}
-
- public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
index c2155ac..61dddb8 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/CsvOutputFormatITCase.java
@@ -17,13 +17,12 @@
package org.apache.flink.test.streaming.api.outputformat;
-import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.util.StreamingProgramTestBase;
import org.apache.flink.test.testdata.WordCountData;
-import org.apache.flink.util.Collector;
+import org.apache.flink.test.testfunctions.Tokenizer;
public class CsvOutputFormatITCase extends StreamingProgramTestBase {
@@ -56,23 +55,5 @@ public class CsvOutputFormatITCase extends StreamingProgramTestBase {
.replaceAll("[\\\\(\\\\)]", ""), resultPath);
}
- public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out)
- throws Exception {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
index 2940e6d..6ea864e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/api/outputformat/TextOutputFormatITCase.java
@@ -22,6 +22,7 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.util.StreamingProgramTestBase;
import org.apache.flink.test.testdata.WordCountData;
+import org.apache.flink.test.testfunctions.Tokenizer;
public class TextOutputFormatITCase extends StreamingProgramTestBase {
@@ -39,7 +40,7 @@ public class TextOutputFormatITCase extends StreamingProgramTestBase {
DataStream<String> text = env.fromElements(WordCountData.TEXT);
DataStream<Tuple2<String, Integer>> counts = text
- .flatMap(new CsvOutputFormatITCase.Tokenizer())
+ .flatMap(new Tokenizer())
.keyBy(0).sum(1);
counts.writeAsText(resultPath);
http://git-wip-us.apache.org/repos/asf/flink/blob/e5e6da0b/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java b/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java
new file mode 100644
index 0000000..9b3764d
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/testfunctions/Tokenizer.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.testfunctions;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+
+public final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
+
+ @Override
+ public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
+ String[] tokens = value.toLowerCase().split("\\W+");
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Tuple2<>(token, 1));
+ }
+ }
+ }
+}