You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2014/11/03 17:04:11 UTC
[3/5] git commit: Remove obsolete collection execution example.
Correct remote collector format example.
Remove obsolete collection execution example.
Correct remote collector format example.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/2557832a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/2557832a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/2557832a
Branch: refs/heads/master
Commit: 2557832af50e5d7bb479d568b370bfdd96b54fef
Parents: a747b61
Author: Stephan Ewen <se...@apache.org>
Authored: Sun Oct 5 21:02:33 2014 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon Nov 3 16:08:13 2014 +0100
----------------------------------------------------------------------
.../CollectionExecutionExample.java | 48 --------
.../RemoteCollectorOutputFormatExample.java | 114 +++++++++++++++++++
.../RemoteCollectorOutputFormatExample.java | 114 -------------------
3 files changed, 114 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2557832a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java
deleted file mode 100644
index 1ce3e7a..0000000
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/environments/CollectionExecutionExample.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.java.environments;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.CollectionEnvironment;
-
-/**
- * This example shows how to use Flink's collection execution functionality.
- * Collection-based execution is an extremely lightweight, non-parallel way to
- * execute programs on small data: The programs are s
- *
- * Because this method of execution spawns no background threads, managed memory,
- * coordinator, or parallel worker, it has a minimal execution footprint.
- */
-public class CollectionExecutionExample {
-
- public static void main(String[] args) throws Exception {
-
- CollectionEnvironment env = new CollectionEnvironment();
-
- env.fromElements("A", "B", "C", "D")
- .map(new MapFunction<String, String>() {
- public String map(String value) {
- return value + " " + 1;
- };
- })
- .print();
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2557832a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/misc/RemoteCollectorOutputFormatExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/misc/RemoteCollectorOutputFormatExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/misc/RemoteCollectorOutputFormatExample.java
new file mode 100644
index 0000000..f524718
--- /dev/null
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/misc/RemoteCollectorOutputFormatExample.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.examples.java.misc;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.aggregation.Aggregations;
+import org.apache.flink.api.java.io.RemoteCollectorConsumer;
+import org.apache.flink.api.java.io.RemoteCollectorImpl;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Collector;
+
+/**
+ * Implements the "WordCount" program that computes a simple word occurrence
+ * histogram over some sample data and collects the results with an
+ * implementation of a {@link RemoteCollectorConsumer}.
+ */
+@SuppressWarnings("serial")
+public class RemoteCollectorOutputFormatExample {
+
+ public static void main(String[] args) throws Exception {
+
+ /**
+ * We create a remote {@link ExecutionEnvironment} here, because this
+ * OutputFormat is designed for use in a distributed setting. For local
+ * use you should consider using the {@link LocalCollectionOutputFormat
+ * <T>}.
+ */
+ final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("<remote>", 6124,
+ "/path/to/your/file.jar");
+
+ // get input data
+ DataSet<String> text = env.fromElements(
+ "To be, or not to be,--that is the question:--",
+ "Whether 'tis nobler in the mind to suffer",
+ "The slings and arrows of outrageous fortune",
+ "Or to take arms against a sea of troubles,");
+
+ DataSet<Tuple2<String, Integer>> counts =
+ // split up the lines in pairs (2-tuples) containing: (word,1)
+ text.flatMap(new LineSplitter())
+ // group by the tuple field "0" and sum up tuple field "1"
+ .groupBy(0).aggregate(Aggregations.SUM, 1);
+
+ // emit result
+ RemoteCollectorImpl.collectLocal(counts,
+ new RemoteCollectorConsumer<Tuple2<String, Integer>>() {
+ // user defined IRemoteCollectorConsumer
+ @Override
+ public void collect(Tuple2<String, Integer> element) {
+ System.out.println("word/occurrences:" + element);
+ }
+ });
+
+ // local collection to store results in
+ Set<Tuple2<String, Integer>> collection = new HashSet<Tuple2<String, Integer>>();
+ // collect results from remote in local collection
+ RemoteCollectorImpl.collectLocal(counts, collection);
+
+ // execute program
+ env.execute("WordCount Example with RemoteCollectorOutputFormat");
+
+ System.out.println(collection);
+
+ RemoteCollectorImpl.shutdownAll();
+ }
+
+ //
+ // User Functions
+ //
+
+ /**
+ * Implements the string tokenizer that splits sentences into words as a
+ * user-defined FlatMapFunction. The function takes a line (String) and
+ * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
+ * Integer>).
+ */
+ public static final class LineSplitter implements
+ FlatMapFunction<String, Tuple2<String, Integer>> {
+
+ @Override
+ public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
+ // normalize and split the line
+ String[] tokens = value.toLowerCase().split("\\W+");
+
+ // emit the pairs
+ for (String token : tokens) {
+ if (token.length() > 0) {
+ out.collect(new Tuple2<String, Integer>(token, 1));
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2557832a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
deleted file mode 100644
index 36b5c82..0000000
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/remotecollectoroutputformat/RemoteCollectorOutputFormatExample.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.java.remotecollectoroutputformat;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.io.RemoteCollectorConsumer;
-import org.apache.flink.api.java.io.RemoteCollectorImpl;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Collector;
-
-/**
- * Implements the "WordCount" program that computes a simple word occurrence
- * histogram over some sample data and collects the results with an
- * implementation of a {@link RemoteCollectorConsumer}.
- */
-@SuppressWarnings("serial")
-public class RemoteCollectorOutputFormatExample {
-
- public static void main(String[] args) throws Exception {
-
- /**
- * We create a remote {@link ExecutionEnvironment} here, because this
- * OutputFormat is designed for use in a distributed setting. For local
- * use you should consider using the {@link LocalCollectionOutputFormat
- * <T>}.
- */
- final ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("<remote>", 6124,
- "/path/to/your/file.jar");
-
- // get input data
- DataSet<String> text = env.fromElements(
- "To be, or not to be,--that is the question:--",
- "Whether 'tis nobler in the mind to suffer",
- "The slings and arrows of outrageous fortune",
- "Or to take arms against a sea of troubles,");
-
- DataSet<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new LineSplitter())
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0).aggregate(Aggregations.SUM, 1);
-
- // emit result
- RemoteCollectorImpl.collectLocal(counts,
- new RemoteCollectorConsumer<Tuple2<String, Integer>>() {
- // user defined IRemoteCollectorConsumer
- @Override
- public void collect(Tuple2<String, Integer> element) {
- System.out.println("word/occurrences:" + element);
- }
- });
-
- // local collection to store results in
- Set<Tuple2<String, Integer>> collection = new HashSet<Tuple2<String, Integer>>();
- // collect results from remote in local collection
- RemoteCollectorImpl.collectLocal(counts, collection);
-
- // execute program
- env.execute("WordCount Example with RemoteCollectorOutputFormat");
-
- System.out.println(collection);
-
- RemoteCollectorImpl.shutdownAll();
- }
-
- //
- // User Functions
- //
-
- /**
- * Implements the string tokenizer that splits sentences into words as a
- * user-defined FlatMapFunction. The function takes a line (String) and
- * splits it into multiple pairs in the form of "(word,1)" (Tuple2<String,
- * Integer>).
- */
- public static final class LineSplitter implements
- FlatMapFunction<String, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
-}
\ No newline at end of file