You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2014/09/20 15:10:47 UTC
[04/18] git commit: [streaming] Reduce operator added to
ConnectedDataStream + test
[streaming] Reduce operator added to ConnectedDataStream + test
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/47dca692
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/47dca692
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/47dca692
Branch: refs/heads/master
Commit: 47dca692e2db320cfde14f4c6905b31972442224
Parents: d97efde
Author: gyfora <gy...@gmail.com>
Authored: Tue Sep 2 13:58:40 2014 +0200
Committer: mbalassi <ba...@gmail.com>
Committed: Sat Sep 20 13:42:04 2014 +0200
----------------------------------------------------------------------
.../api/datastream/ConnectedDataStream.java | 55 +++++++++++----
.../datastream/GroupedConnectedDataStream.java | 17 ++---
.../operator/co/CoGroupReduceInvokable.java | 2 +-
.../operator/co/CoReduceInvokable.java | 70 +++++++++++++++++++
.../operator/co/CoStreamReduceInvokable.java | 70 -------------------
.../api/invokable/operator/CoReduceTest.java | 71 ++++++++++++++++++++
6 files changed, 193 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/47dca692/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 920278c..4f17ceb 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -27,10 +27,12 @@ import org.apache.flink.streaming.api.JobGraphBuilder;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
import org.apache.flink.streaming.api.function.co.CoMapFunction;
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.function.co.RichCoMapFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
+import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
import org.apache.flink.streaming.util.serialization.FunctionTypeWrapper;
import org.apache.flink.streaming.util.serialization.TypeSerializerWrapper;
@@ -102,12 +104,13 @@ public class ConnectedDataStream<IN1, IN2> {
}
/**
- * Applies a CoMap transformation on two separate {@link DataStream}s. The
- * transformation calls a {@link CoMapFunction#map1} for each element of the
- * first input and {@link CoMapFunction#map2} for each element of the second
- * input. Each CoMapFunction call returns exactly one element. The user can
- * also extend {@link RichCoMapFunction} to gain access to other features
- * provided by the {@link RichFuntion} interface.
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * {@link CoMapFunction#map1} for each element of the first input and
+ * {@link CoMapFunction#map2} for each element of the second input. Each
+ * CoMapFunction call returns exactly one element. The user can also extend
+ * {@link RichCoMapFunction} to gain access to other features provided by
+ * the {@link RichFuntion} interface.
*
* @param coMapper
* The CoMapFunction used to jointly transform the two input
@@ -127,13 +130,13 @@ public class ConnectedDataStream<IN1, IN2> {
}
/**
- * Applies a CoFlatMap transformation on two separate {@link DataStream}s.
- * The transformation calls a {@link CoFlatMapFunction#map1} for each
- * element of the first input and {@link CoFlatMapFunction#map2} for each
- * element of the second input. Each CoFlatMapFunction call returns any
- * number of elements including none. The user can also extend
- * {@link RichFlatMapFunction} to gain access to other features provided by
- * the {@link RichFuntion} interface.
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * {@link CoFlatMapFunction#map1} for each element of the first input and
+ * {@link CoFlatMapFunction#map2} for each element of the second input. Each
+ * CoFlatMapFunction call returns any number of elements including none. The
+ * user can also extend {@link RichFlatMapFunction} to gain access to other
+ * features provided by the {@link RichFuntion} interface.
*
* @param coFlatMapper
* The CoFlatMapFunction used to jointly transform the two input
@@ -153,6 +156,32 @@ public class ConnectedDataStream<IN1, IN2> {
outTypeWrapper, new CoFlatMapInvokable<IN1, IN2, OUT>(coFlatMapper));
}
+ /**
+ * Applies a reduce transformation on both input of a
+ * {@link ConnectedDataStream} and maps the output to a common type. The
+ * transformation calls {@link CoReduceFunction#reduce1} and
+ * {@link CoReduceFunction#map1} for each element of the first input and
+ * {@link CoReduceFunction#reduce2} and {@link CoReduceFunction#map2} for
+ * each element of the second input.
+ *
+ * @param coReducer
+ * The {@link CoReduceFunction} that will be called for every
+ * element of the inputs.
+ * @return The transformed DataStream.
+ */
+ public <OUT> SingleOutputStreamOperator<OUT, ?> reduce(CoReduceFunction<IN1, IN2, OUT> coReducer) {
+
+ FunctionTypeWrapper<IN1> in1TypeWrapper = new FunctionTypeWrapper<IN1>(coReducer,
+ CoReduceFunction.class, 0);
+ FunctionTypeWrapper<IN2> in2TypeWrapper = new FunctionTypeWrapper<IN2>(coReducer,
+ CoReduceFunction.class, 1);
+ FunctionTypeWrapper<OUT> outTypeWrapper = new FunctionTypeWrapper<OUT>(coReducer,
+ CoReduceFunction.class, 2);
+
+ return addCoFunction("coReduce", coReducer, in1TypeWrapper, in2TypeWrapper, outTypeWrapper,
+ new CoReduceInvokable<IN1, IN2, OUT>(coReducer));
+ }
+
protected <OUT> SingleOutputStreamOperator<OUT, ?> addCoFunction(String functionName,
final Function function, TypeSerializerWrapper<IN1> in1TypeWrapper,
TypeSerializerWrapper<IN2> in2TypeWrapper, TypeSerializerWrapper<OUT> outTypeWrapper,
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/47dca692/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedConnectedDataStream.java
index 13ba3ab..8276561 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedConnectedDataStream.java
@@ -37,16 +37,17 @@ public class GroupedConnectedDataStream<IN1, IN2> extends ConnectedDataStream<IN
}
/**
- * Applies a CoReduce transformation on the grouped data stream grouped on
- * by the given key position. The {@link CoReduceFunction} will receive
- * input values based on the key positions. The transformation calls
- * {@link CoReduceFunction#reduce1} and {@link CoReduceFunction#map1} for
- * each element of the first input and {@link CoReduceFunction#reduce2} and
- * {@link CoReduceFunction#map2} for each element of the second input. For
- * each input, only values with the same key will go to the same reducer.
+ * Applies a CoReduce transformation on a {@link ConnectedDataStream}
+ * grouped by the given key position and maps the output to a common type.
+ * The {@link CoReduceFunction} will receive input values based on the key
+ * positions. The transformation calls {@link CoReduceFunction#reduce1} and
+ * {@link CoReduceFunction#map1} for each element of the first input and
+ * {@link CoReduceFunction#reduce2} and {@link CoReduceFunction#map2} for
+ * each element of the second input. For each input, only values with the
+ * same key will go to the same reducer.
*
* @param coReducer
- * The {@link CoReduceFunction} that will be called for every two
+ * The {@link CoReduceFunction} that will be called for every
* element with the same key of each input DataStream.
* @return The transformed DataStream.
*/
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/47dca692/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupReduceInvokable.java
index d7ad32e..ab827d8 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupReduceInvokable.java
@@ -20,7 +20,7 @@ package org.apache.flink.streaming.api.invokable.operator.co;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.state.MutableTableState;
-public class CoGroupReduceInvokable<IN1, IN2, OUT> extends CoStreamReduceInvokable<IN1, IN2, OUT> {
+public class CoGroupReduceInvokable<IN1, IN2, OUT> extends CoReduceInvokable<IN1, IN2, OUT> {
private static final long serialVersionUID = 1L;
private int keyPosition1;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/47dca692/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
new file mode 100644
index 0000000..407f217
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoReduceInvokable.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+
+public class CoReduceInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
+ private static final long serialVersionUID = 1L;
+
+ protected CoReduceFunction<IN1, IN2, OUT> coReducer;
+ protected IN1 currentValue1 = null;
+ protected IN2 currentValue2 = null;
+ protected IN1 nextValue1 = null;
+ protected IN2 nextValue2 = null;
+
+ public CoReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer) {
+ super(coReducer);
+ this.coReducer = coReducer;
+ currentValue1 = null;
+ currentValue2 = null;
+ }
+
+ @Override
+ public void handleStream1() throws Exception {
+ nextValue1 = reuse1.getObject();
+ callUserFunctionAndLogException1();
+ }
+
+ @Override
+ public void handleStream2() throws Exception {
+ nextValue2 = reuse2.getObject();
+ callUserFunctionAndLogException2();
+ }
+
+ @Override
+ protected void callUserFunction1() throws Exception {
+ if (currentValue1 != null) {
+ currentValue1 = coReducer.reduce1(currentValue1, nextValue1);
+ } else {
+ currentValue1 = nextValue1;
+ }
+ collector.collect(coReducer.map1(currentValue1));
+ }
+
+ @Override
+ protected void callUserFunction2() throws Exception {
+ if (currentValue2 != null) {
+ currentValue2 = coReducer.reduce2(currentValue2, nextValue2);
+ } else {
+ currentValue2 = nextValue2;
+ }
+ collector.collect(coReducer.map2(currentValue2));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/47dca692/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceInvokable.java
deleted file mode 100644
index e4acb99..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceInvokable.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-
-public abstract class CoStreamReduceInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
-
- protected CoReduceFunction<IN1, IN2, OUT> coReducer;
- protected IN1 currentValue1 = null;
- protected IN2 currentValue2 = null;
- protected IN1 nextValue1 = null;
- protected IN2 nextValue2 = null;
-
- public CoStreamReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer) {
- super(coReducer);
- this.coReducer = coReducer;
- currentValue1 = null;
- currentValue2 = null;
- }
-
- @Override
- public void handleStream1() throws Exception {
- nextValue1 = reuse1.getObject();
- callUserFunctionAndLogException1();
- }
-
- @Override
- public void handleStream2() throws Exception {
- nextValue2 = reuse2.getObject();
- callUserFunctionAndLogException2();
- }
-
- @Override
- protected void callUserFunction1() throws Exception {
- if (currentValue1 != null) {
- currentValue1 = coReducer.reduce1(currentValue1, nextValue1);
- } else {
- currentValue1 = nextValue1;
- }
- collector.collect(coReducer.map1(currentValue1));
- }
-
- @Override
- protected void callUserFunction2() throws Exception {
- if (currentValue2 != null) {
- currentValue2 = coReducer.reduce2(currentValue2, nextValue2);
- } else {
- currentValue2 = nextValue2;
- }
- collector.collect(coReducer.map2(nextValue2));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/47dca692/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoReduceTest.java
new file mode 100755
index 0000000..4f52444
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoReduceTest.java
@@ -0,0 +1,71 @@
+/** Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
+import org.apache.flink.streaming.util.MockCoInvokable;
+import org.junit.Test;
+
+public class CoReduceTest {
+
+ public static class MyCoReduceFunction implements CoReduceFunction<Integer, String, Integer> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce1(Integer value1, Integer value2) {
+ return value1 * value2;
+ }
+
+ @Override
+ public String reduce2(String value1, String value2) {
+ return value1 + value2;
+ }
+
+ @Override
+ public Integer map1(Integer value) {
+ return value;
+ }
+
+ @Override
+ public Integer map2(String value) {
+ return Integer.parseInt(value);
+ }
+
+ }
+
+ @Test
+ public void coGroupReduceTest() {
+
+ CoReduceInvokable<Integer, String, Integer> coReduce = new CoReduceInvokable<Integer, String, Integer>(
+ new MyCoReduceFunction());
+
+ List<Integer> expected1 = Arrays.asList(1, 9, 2, 99, 6, 998, 24);
+
+ assertEquals(
+ expected1,
+ (MockCoInvokable.createAndExecute(coReduce, Arrays.asList(1, 2, 3, 4),
+ Arrays.asList("9", "9", "8"))));
+
+ }
+}