You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/01/07 15:12:40 UTC
[01/36] flink git commit: [streaming] Time trigger preNotify fix
Repository: flink
Updated Branches:
refs/heads/release-0.8 2467f36c8 -> 242ff64e0
[streaming] Time trigger preNotify fix
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b2271bd9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b2271bd9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b2271bd9
Branch: refs/heads/release-0.8
Commit: b2271bd9eb3adc1770f40d452ed7fb69614ea649
Parents: 2467f36
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Jan 2 18:33:46 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Sun Jan 4 22:11:13 2015 +0100
----------------------------------------------------------------------
.../streaming/api/windowing/policy/TimeTriggerPolicy.java | 6 +++---
.../api/windowing/policy/TimeTriggerPolicyTest.java | 8 +++-----
2 files changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b2271bd9/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
index 3539ad6..57bccf2 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -107,9 +107,9 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
LinkedList<Object> fakeElements = new LinkedList<Object>();
// check if there is more then one window border missed
// use > here. In case >= would fit, the regular call will do the job.
- while (timestamp.getTimestamp(datapoint) > startTime + granularity) {
+ while (timestamp.getTimestamp(datapoint) >= startTime + granularity) {
startTime += granularity;
- fakeElements.add(startTime-1);
+ fakeElements.add(startTime - 1);
}
return (Object[]) fakeElements.toArray();
}
@@ -146,7 +146,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
// start time is excluded, but end time is included: >=
if (System.currentTimeMillis() >= startTime + granularity) {
startTime += granularity;
- callback.sendFakeElement(startTime-1);
+ callback.sendFakeElement(startTime - 1);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/b2271bd9/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
index 28d35ec..9c77a55 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
@@ -50,8 +50,7 @@ public class TimeTriggerPolicyTest {
// test different granularity
for (long granularity = 0; granularity < 31; granularity++) {
// create policy
- TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity,
- timeStamp);
+ TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity, timeStamp);
// remember window border
// Remark: This might NOT work in case the timeStamp uses
@@ -101,11 +100,10 @@ public class TimeTriggerPolicyTest {
};
// create policy
- TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5,
- timeStamp);
+ TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5, timeStamp);
// expected result
- Long[][] result = { {}, {}, { 4L, 9L, 14L }, { 24L } };
+ Long[][] result = { {}, {}, { 4L, 9L, 14L, 19L }, { 24L } };
// call policy
for (int i = 0; i < times.length; i++) {
[06/36] flink git commit: [scala] [streaming] Added SplitDataStream
functionality
Posted by mb...@apache.org.
[scala] [streaming] Added SplitDataStream functionality
Conflicts:
flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a4c4f26e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a4c4f26e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a4c4f26e
Branch: refs/heads/release-0.8
Commit: a4c4f26e70088ed5cd5bf8a8f01585f09674cf04
Parents: e0ab489
Author: mbalassi <mb...@apache.org>
Authored: Mon Dec 15 14:15:35 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:53:28 2015 +0100
----------------------------------------------------------------------
.../api/datastream/SplitDataStream.java | 4 +-
.../flink/api/scala/streaming/DataStream.scala | 33 ++++++++++++-
.../api/scala/streaming/SplitDataStream.scala | 49 ++++++++++++++++++++
3 files changed, 83 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a4c4f26e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
index 5a8f038..4fac04c 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SplitDataStream.java
@@ -70,9 +70,9 @@ public class SplitDataStream<OUT> {
return returnStream;
}
- private DataStream<OUT> selectOutput(String[] outputName) {
+ private DataStream<OUT> selectOutput(String[] outputNames) {
DataStream<OUT> returnStream = dataStream.copy();
- returnStream.userDefinedNames = Arrays.asList(outputName);
+ returnStream.userDefinedNames = Arrays.asList(outputNames);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a4c4f26e/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 69b8359..f0932d8 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -41,7 +41,9 @@ import org.apache.flink.streaming.api.function.sink.SinkFunction
import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy
+import org.apache.flink.streaming.api.collector.OutputSelector
import scala.collection.JavaConversions._
+import java.util.HashMap
class DataStream[T](javaStream: JavaStream[T]) {
@@ -423,7 +425,36 @@ class DataStream[T](javaStream: JavaStream[T]) {
def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]): WindowedDataStream[T] = new WindowedDataStream[T](javaStream.window(triggers, evicters))
/**
- * >>>>>>> 12178aa... [scala] [streaming] Windowing functionality added to scala api
+ *
+ * Operator used for directing tuples to specific named outputs using an
+ * OutputSelector. Calling this method on an operator creates a new
+ * SplitDataStream.
+ */
+ def split(selector: OutputSelector[T]): SplitDataStream[T] = javaStream match {
+ case op: SingleOutputStreamOperator[_, _] => new SplitDataStream[T](op.split(selector))
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " can not be " +
+ "split.")
+ }
+
+ /**
+ * Creates a new SplitDataStream that contains only the elements satisfying the
+ * given output selector predicate.
+ */
+ def split(fun: T => TraversableOnce[String]): SplitDataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("OutputSelector must not be null.")
+ }
+ val selector = new OutputSelector[T] {
+ val cleanFun = clean(fun)
+ def select(in: T): java.lang.Iterable[String] = {
+ asJavaIterable(cleanFun(in).toIterable)
+ }
+ }
+ split(selector)
+ }
+
+ /**
* Writes a DataStream to the standard output stream (stdout). For each
* element of the DataStream the result of .toString is
* written.
http://git-wip-us.apache.org/repos/asf/flink/blob/a4c4f26e/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
new file mode 100644
index 0000000..0b0cce5
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+
+/**
+ * The SplitDataStream represents an operator that has been split using an
+ * {@link OutputSelector}. Named outputs can be selected using the
+ * {@link #select} function.
+ *
+ * @param <OUT>
+ * The type of the output.
+ */
+class SplitDataStream[T](javaStream: SplitJavaStream[T]) {
+
+ /**
+ * Gets the underlying java DataStream object.
+ */
+ private[flink] def getJavaStream: SplitJavaStream[T] = javaStream
+
+ /**
+ * Sets the output names for which the next operator will receive values.
+ */
+ def select(outputNames: String*): DataStream[T] =
+ new DataStream[T](javaStream.select(outputNames: _*))
+
+ /**
+ * Selects all output names from a split data stream.
+ */
+ def selectAll(): DataStream[T] = new DataStream[T](javaStream.selectAll())
+
+}
\ No newline at end of file
[02/36] flink git commit: [scala] [streaming] Base functionality
added for streaming scala api
Posted by mb...@apache.org.
[scala] [streaming] Base functionality added for streaming scala api
Conflicts:
flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6b1fd156
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6b1fd156
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6b1fd156
Branch: refs/heads/release-0.8
Commit: 6b1fd156a774a8292dd2e9227d611dcca5b9c526
Parents: b2271bd
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Dec 11 15:22:03 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:49:00 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 37 ++---
.../flink/streaming/api/StreamConfig.java | 16 --
.../api/datastream/ConnectedDataStream.java | 47 +++---
.../streaming/api/datastream/DataStream.java | 70 ++++----
.../api/datastream/GroupedDataStream.java | 10 +-
.../api/datastream/StreamProjection.java | 161 ++++++++-----------
.../api/datastream/WindowedDataStream.java | 14 +-
.../environment/StreamExecutionEnvironment.java | 38 ++---
.../function/source/FromElementsFunction.java | 90 ++++++-----
.../api/streamvertex/StreamVertex.java | 3 +-
flink-scala/pom.xml | 6 +
.../flink/api/scala/streaming/DataStream.scala | 76 +++++++++
.../streaming/StreamExecutionEnvironment.scala | 150 +++++++++++++++++
13 files changed, 425 insertions(+), 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index d66e388..f358de9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -72,7 +72,6 @@ public class JobGraphBuilder {
private Map<String, StreamRecordSerializer<?>> typeSerializersIn2;
private Map<String, StreamRecordSerializer<?>> typeSerializersOut1;
private Map<String, StreamRecordSerializer<?>> typeSerializersOut2;
- private Map<String, byte[]> serializedFunctions;
private Map<String, byte[]> outputSelectors;
private Map<String, Class<? extends AbstractInvokable>> vertexClasses;
private Map<String, Integer> iterationIds;
@@ -104,7 +103,6 @@ public class JobGraphBuilder {
typeSerializersIn2 = new HashMap<String, StreamRecordSerializer<?>>();
typeSerializersOut1 = new HashMap<String, StreamRecordSerializer<?>>();
typeSerializersOut2 = new HashMap<String, StreamRecordSerializer<?>>();
- serializedFunctions = new HashMap<String, byte[]>();
outputSelectors = new HashMap<String, byte[]>();
vertexClasses = new HashMap<String, Class<? extends AbstractInvokable>>();
iterationIds = new HashMap<String, Integer>();
@@ -133,18 +131,14 @@ public class JobGraphBuilder {
* Output type for serialization
* @param operatorName
* Operator type
- * @param serializedFunction
- * Serialized udf
* @param parallelism
* Number of parallel instances created
*/
public <IN, OUT> void addStreamVertex(String vertexName,
StreamInvokable<IN, OUT> invokableObject, TypeInformation<IN> inTypeInfo,
- TypeInformation<OUT> outTypeInfo, String operatorName, byte[] serializedFunction,
- int parallelism) {
+ TypeInformation<OUT> outTypeInfo, String operatorName, int parallelism) {
- addVertex(vertexName, StreamVertex.class, invokableObject, operatorName,
- serializedFunction, parallelism);
+ addVertex(vertexName, StreamVertex.class, invokableObject, operatorName, parallelism);
StreamRecordSerializer<IN> inSerializer = inTypeInfo != null ? new StreamRecordSerializer<IN>(
inTypeInfo) : null;
@@ -171,8 +165,6 @@ public class JobGraphBuilder {
* Output type for serialization
* @param operatorName
* Operator type
- * @param serializedFunction
- * Serialized udf
* @param parallelism
* Number of parallel instances created
*/
@@ -185,7 +177,7 @@ public class JobGraphBuilder {
function);
addStreamVertex(vertexName, invokableObject, inTypeInfo, outTypeInfo, operatorName,
- serializedFunction, parallelism);
+ parallelism);
}
/**
@@ -206,7 +198,7 @@ public class JobGraphBuilder {
public void addIterationHead(String vertexName, String iterationHead, Integer iterationID,
int parallelism, long waitTime) {
- addVertex(vertexName, StreamIterationHead.class, null, null, null, parallelism);
+ addVertex(vertexName, StreamIterationHead.class, null, null, parallelism);
iterationIds.put(vertexName, iterationID);
iterationIDtoHeadName.put(iterationID, vertexName);
@@ -247,7 +239,7 @@ public class JobGraphBuilder {
throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
}
- addVertex(vertexName, StreamIterationTail.class, null, null, null, parallelism);
+ addVertex(vertexName, StreamIterationTail.class, null, null, parallelism);
iterationIds.put(vertexName, iterationID);
iterationIDtoTailName.put(iterationID, vertexName);
@@ -264,10 +256,9 @@ public class JobGraphBuilder {
public <IN1, IN2, OUT> void addCoTask(String vertexName,
CoInvokable<IN1, IN2, OUT> taskInvokableObject, TypeInformation<IN1> in1TypeInfo,
TypeInformation<IN2> in2TypeInfo, TypeInformation<OUT> outTypeInfo,
- String operatorName, byte[] serializedFunction, int parallelism) {
+ String operatorName, int parallelism) {
- addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName,
- serializedFunction, parallelism);
+ addVertex(vertexName, CoStreamVertex.class, taskInvokableObject, operatorName, parallelism);
addTypeSerializers(vertexName, new StreamRecordSerializer<IN1>(in1TypeInfo),
new StreamRecordSerializer<IN2>(in2TypeInfo), new StreamRecordSerializer<OUT>(
@@ -289,20 +280,16 @@ public class JobGraphBuilder {
* The user defined invokable object
* @param operatorName
* Type of the user defined operator
- * @param serializedFunction
- * Serialized operator
* @param parallelism
* Number of parallel instances created
*/
private void addVertex(String vertexName, Class<? extends AbstractInvokable> vertexClass,
- StreamInvokable<?, ?> invokableObject, String operatorName, byte[] serializedFunction,
- int parallelism) {
+ StreamInvokable<?, ?> invokableObject, String operatorName, int parallelism) {
vertexClasses.put(vertexName, vertexClass);
setParallelism(vertexName, parallelism);
invokableObjects.put(vertexName, invokableObject);
operatorNames.put(vertexName, operatorName);
- serializedFunctions.put(vertexName, serializedFunction);
outEdgeList.put(vertexName, new ArrayList<String>());
outEdgeType.put(vertexName, new ArrayList<Integer>());
outEdgeNames.put(vertexName, new ArrayList<List<String>>());
@@ -333,8 +320,6 @@ public class JobGraphBuilder {
// Get vertex attributes
Class<? extends AbstractInvokable> vertexClass = vertexClasses.get(vertexName);
StreamInvokable<?, ?> invokableObject = invokableObjects.get(vertexName);
- String operatorName = operatorNames.get(vertexName);
- byte[] serializedFunction = serializedFunctions.get(vertexName);
int parallelism = vertexParallelism.get(vertexName);
byte[] outputSelector = outputSelectors.get(vertexName);
Map<String, OperatorState<?>> state = operatorStates.get(vertexName);
@@ -362,7 +347,6 @@ public class JobGraphBuilder {
// Set vertex config
config.setUserInvokable(invokableObject);
config.setVertexName(vertexName);
- config.setFunction(serializedFunction, operatorName);
config.setOutputSelector(outputSelector);
config.setOperatorStates(state);
@@ -522,8 +506,8 @@ public class JobGraphBuilder {
}
/**
- * Sets udf operator and TypeSerializerWrapper from one vertex to another,
- * used with some sinks.
+ * Sets TypeSerializerWrapper from one vertex to another, used with some
+ * sinks.
*
* @param from
* from
@@ -532,7 +516,6 @@ public class JobGraphBuilder {
*/
public void setBytesFrom(String from, String to) {
operatorNames.put(to, operatorNames.get(from));
- serializedFunctions.put(to, serializedFunctions.get(from));
typeSerializersIn1.put(to, typeSerializersOut1.get(from));
typeSerializersIn2.put(to, typeSerializersOut2.get(from));
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
index 9800b63..8837b85 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/StreamConfig.java
@@ -47,7 +47,6 @@ public class StreamConfig {
private static final String OUTPUT_SELECTOR = "outputSelector";
private static final String DIRECTED_EMIT = "directedEmit";
private static final String FUNCTION_NAME = "operatorName";
- private static final String FUNCTION = "operator";
private static final String VERTEX_NAME = "vertexName";
private static final String SERIALIZEDUDF = "serializedudf";
private static final String USER_FUNCTION = "userfunction";
@@ -173,21 +172,6 @@ public class StreamConfig {
return config.getString(VERTEX_NAME, null);
}
- public void setFunction(byte[] serializedFunction, String functionName) {
- if (serializedFunction != null) {
- config.setBytes(FUNCTION, serializedFunction);
- config.setString(FUNCTION_NAME, functionName);
- }
- }
-
- public Object getFunction(ClassLoader cl) {
- try {
- return InstantiationUtil.readObjectFromConfig(this.config, FUNCTION, cl);
- } catch (Exception e) {
- throw new RuntimeException("Cannot deserialize invokable object", e);
- }
- }
-
public String getFunctionName() {
return config.getString(FUNCTION_NAME, "");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index dcc3dab..c663315 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -17,13 +17,9 @@
package org.apache.flink.streaming.api.datastream;
-import java.io.Serializable;
import java.util.List;
-import org.apache.commons.lang3.SerializationException;
-import org.apache.commons.lang3.SerializationUtils;
import org.apache.flink.api.common.functions.CrossFunction;
-import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.ClosureCleaner;
@@ -53,7 +49,7 @@ import org.apache.flink.util.Collector;
* The ConnectedDataStream represents a stream for two different data types. It
* can be used to apply transformations like {@link CoMapFunction} on two
* {@link DataStream}s
- *
+ *
* @param <IN1>
* Type of the first input data steam.
* @param <IN2>
@@ -417,8 +413,9 @@ public class ConnectedDataStream<IN1, IN2> {
TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoMapFunction.class,
coMapper.getClass(), 2, null, null);
- return addCoFunction("coMap", clean(coMapper), outTypeInfo,
- new CoMapInvokable<IN1, IN2, OUT>(clean(coMapper)));
+ return addCoFunction("coMap", outTypeInfo, new CoMapInvokable<IN1, IN2, OUT>(
+ clean(coMapper)));
+
}
/**
@@ -441,8 +438,8 @@ public class ConnectedDataStream<IN1, IN2> {
TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoFlatMapFunction.class,
coFlatMapper.getClass(), 2, null, null);
- return addCoFunction("coFlatMap", clean(coFlatMapper), outTypeInfo,
- new CoFlatMapInvokable<IN1, IN2, OUT>(clean(coFlatMapper)));
+ return addCoFunction("coFlatMap", outTypeInfo, new CoFlatMapInvokable<IN1, IN2, OUT>(
+ clean(coFlatMapper)));
}
/**
@@ -466,8 +463,8 @@ public class ConnectedDataStream<IN1, IN2> {
TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoReduceFunction.class,
coReducer.getClass(), 2, null, null);
- return addCoFunction("coReduce", clean(coReducer), outTypeInfo,
- getReduceInvokable(clean(coReducer)));
+ return addCoFunction("coReduce", outTypeInfo, getReduceInvokable(clean(coReducer)));
+
}
/**
@@ -531,9 +528,9 @@ public class ConnectedDataStream<IN1, IN2> {
TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CoWindowFunction.class,
coWindowFunction.getClass(), 2, null, null);
- return addCoFunction("coWindowReduce", clean(coWindowFunction), outTypeInfo,
- new CoWindowInvokable<IN1, IN2, OUT>(clean(coWindowFunction), windowSize, slideInterval,
- timestamp1, timestamp2));
+ return addCoFunction("coWindowReduce", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
+ clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
+
}
protected <OUT> CoInvokable<IN1, IN2, OUT> getReduceInvokable(
@@ -607,27 +604,21 @@ public class ConnectedDataStream<IN1, IN2> {
throw new IllegalArgumentException("Slide interval must be positive");
}
- return addCoFunction("coWindowReduce", clean(coWindowFunction), outTypeInfo,
- new CoWindowInvokable<IN1, IN2, OUT>(clean(coWindowFunction), windowSize, slideInterval,
- timestamp1, timestamp2));
+ return addCoFunction("coWindowReduce", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
+ clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
+
}
- protected <OUT> SingleOutputStreamOperator<OUT, ?> addCoFunction(String functionName,
- final Function function, TypeInformation<OUT> outTypeInfo,
- CoInvokable<IN1, IN2, OUT> functionInvokable) {
+ public <OUT> SingleOutputStreamOperator<OUT, ?> addCoFunction(String functionName,
+ TypeInformation<OUT> outTypeInfo, CoInvokable<IN1, IN2, OUT> functionInvokable) {
@SuppressWarnings({ "unchecked", "rawtypes" })
SingleOutputStreamOperator<OUT, ?> returnStream = new SingleOutputStreamOperator(
environment, functionName, outTypeInfo);
- try {
- dataStream1.jobGraphBuilder.addCoTask(returnStream.getId(), functionInvokable,
- getInputType1(), getInputType2(), outTypeInfo, functionName,
- SerializationUtils.serialize((Serializable) function),
- environment.getDegreeOfParallelism());
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize user defined function");
- }
+ dataStream1.jobGraphBuilder.addCoTask(returnStream.getId(), functionInvokable,
+ getInputType1(), getInputType2(), outTypeInfo, functionName,
+ environment.getDegreeOfParallelism());
dataStream1.connectGraph(dataStream1, returnStream.getId(), 1);
dataStream1.connectGraph(dataStream2, returnStream.getId(), 2);
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 6e8da0a..04929c1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -17,15 +17,11 @@
package org.apache.flink.streaming.api.datastream;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.lang3.SerializationException;
-import org.apache.commons.lang3.SerializationUtils;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.functions.RichFilterFunction;
@@ -401,8 +397,7 @@ public class DataStream<OUT> {
TypeInformation<R> outType = TypeExtractor.getMapReturnTypes(clean(mapper), getType());
- return addFunction("map", clean(mapper), getType(), outType, new MapInvokable<OUT, R>(
- clean(mapper)));
+ return transform("map", outType, new MapInvokable<OUT, R>(clean(mapper)));
}
/**
@@ -423,10 +418,11 @@ public class DataStream<OUT> {
*/
public <R> SingleOutputStreamOperator<R, ?> flatMap(FlatMapFunction<OUT, R> flatMapper) {
- TypeInformation<R> outType = TypeExtractor.getFlatMapReturnTypes(clean(flatMapper), getType());
+ TypeInformation<R> outType = TypeExtractor.getFlatMapReturnTypes(clean(flatMapper),
+ getType());
+
+ return transform("flatMap", outType, new FlatMapInvokable<OUT, R>(clean(flatMapper)));
- return addFunction("flatMap", clean(flatMapper), getType(), outType,
- new FlatMapInvokable<OUT, R>(clean(flatMapper)));
}
/**
@@ -442,8 +438,8 @@ public class DataStream<OUT> {
*/
public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reducer) {
- return addFunction("reduce", clean(reducer), getType(), getType(),
- new StreamReduceInvokable<OUT>(clean(reducer)));
+ return transform("reduce", getType(), new StreamReduceInvokable<OUT>(clean(reducer)));
+
}
/**
@@ -461,8 +457,8 @@ public class DataStream<OUT> {
* @return The filtered DataStream.
*/
public SingleOutputStreamOperator<OUT, ?> filter(FilterFunction<OUT> filter) {
- return addFunction("filter", clean(filter), getType(), getType(), new FilterInvokable<OUT>(clean(
- filter)));
+ return transform("filter", getType(), new FilterInvokable<OUT>(clean(filter)));
+
}
/**
@@ -742,7 +738,7 @@ public class DataStream<OUT> {
public SingleOutputStreamOperator<Long, ?> count() {
TypeInformation<Long> outTypeInfo = TypeExtractor.getForObject(Long.valueOf(0));
- return addFunction("counter", null, getType(), outTypeInfo, new CounterInvokable<OUT>());
+ return transform("counter", outTypeInfo, new CounterInvokable<OUT>());
}
/**
@@ -1120,8 +1116,7 @@ public class DataStream<OUT> {
StreamReduceInvokable<OUT> invokable = new StreamReduceInvokable<OUT>(aggregate);
- SingleOutputStreamOperator<OUT, ?> returnStream = addFunction("reduce", clean(aggregate),
- typeInfo, typeInfo, invokable);
+ SingleOutputStreamOperator<OUT, ?> returnStream = transform("reduce", typeInfo, invokable);
return returnStream;
}
@@ -1137,34 +1132,28 @@ public class DataStream<OUT> {
}
/**
- * Internal function for passing the user defined functions to the JobGraph
- * of the job.
- *
- * @param functionName
- * name of the function
- * @param function
- * the user defined function
- * @param functionInvokable
- * the wrapping JobVertex instance
+ * Method for passing user defined invokables along with the type
+ * informations that will transform the DataStream.
+ *
+ * @param operatorName
+ * name of the operator, for logging purposes
+ * @param outTypeInfo
+ * the output type of the operator
+ * @param invokable
+ * the object containing the transformation logic
* @param <R>
* type of the return stream
* @return the data stream constructed
*/
- protected <R> SingleOutputStreamOperator<R, ?> addFunction(String functionName,
- final Function function, TypeInformation<OUT> inTypeInfo,
- TypeInformation<R> outTypeInfo, StreamInvokable<OUT, R> functionInvokable) {
+ public <R> SingleOutputStreamOperator<R, ?> transform(String operatorName,
+ TypeInformation<R> outTypeInfo, StreamInvokable<OUT, R> invokable) {
DataStream<OUT> inputStream = this.copy();
@SuppressWarnings({ "unchecked", "rawtypes" })
SingleOutputStreamOperator<R, ?> returnStream = new SingleOutputStreamOperator(environment,
- functionName, outTypeInfo);
+ operatorName, outTypeInfo);
- try {
- jobGraphBuilder.addStreamVertex(returnStream.getId(), functionInvokable, inTypeInfo,
- outTypeInfo, functionName,
- SerializationUtils.serialize((Serializable) function), degreeOfParallelism);
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize user defined function");
- }
+ jobGraphBuilder.addStreamVertex(returnStream.getId(), invokable, getType(), outTypeInfo,
+ operatorName, degreeOfParallelism);
connectGraph(inputStream, returnStream.getId(), 0);
@@ -1235,13 +1224,8 @@ public class DataStream<OUT> {
SinkFunction<OUT> sinkFunction, TypeInformation<OUT> inTypeInfo) {
DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", typeInfo);
- try {
- jobGraphBuilder.addStreamVertex(returnStream.getId(), new SinkInvokable<OUT>(
- clean(sinkFunction)), inTypeInfo, null, "sink", SerializationUtils
- .serialize(clean(sinkFunction)), degreeOfParallelism);
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize SinkFunction");
- }
+ jobGraphBuilder.addStreamVertex(returnStream.getId(), new SinkInvokable<OUT>(
+ clean(sinkFunction)), inTypeInfo, null, "sink", degreeOfParallelism);
inputStream.connectGraph(inputStream.copy(), returnStream.getId(), 0);
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
index 2620d3e..a2c0f89 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
@@ -29,7 +29,7 @@ import org.apache.flink.streaming.partitioner.StreamPartitioner;
* partitioned by the given {@link KeySelector}. Operators like {@link #reduce},
* {@link #batchReduce} etc. can be applied on the {@link GroupedDataStream} to
* get additional functionality by the grouping.
- *
+ *
* @param <OUT>
* The output type of the {@link GroupedDataStream}.
*/
@@ -62,8 +62,8 @@ public class GroupedDataStream<OUT> extends DataStream<OUT> {
*/
@Override
public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reducer) {
- return addFunction("groupReduce", clean(reducer), getType(), getType(),
- new GroupedReduceInvokable<OUT>(clean(reducer), keySelector));
+ return transform("groupReduce", getType(), new GroupedReduceInvokable<OUT>(clean(reducer),
+ keySelector));
}
/**
@@ -182,8 +182,8 @@ public class GroupedDataStream<OUT> extends DataStream<OUT> {
GroupedReduceInvokable<OUT> invokable = new GroupedReduceInvokable<OUT>(clean(aggregate),
keySelector);
- SingleOutputStreamOperator<OUT, ?> returnStream = addFunction("groupReduce", clean(aggregate),
- typeInfo, typeInfo, invokable);
+ SingleOutputStreamOperator<OUT, ?> returnStream = transform("groupReduce", typeInfo,
+ invokable);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
index cc5f66e..e71b18c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
@@ -83,8 +83,8 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple1<T0>> outType = (TypeInformation<Tuple1<T0>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
- new ProjectInvokable<IN, Tuple1<T0>>(fieldIndexes, outType));
+ return dataStream.transform("projection", outType, new ProjectInvokable<IN, Tuple1<T0>>(
+ fieldIndexes, outType));
}
/**
@@ -111,7 +111,7 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple2<T0, T1>> outType = (TypeInformation<Tuple2<T0, T1>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple2<T0, T1>>(fieldIndexes, outType));
}
@@ -141,7 +141,7 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple3<T0, T1, T2>> outType = (TypeInformation<Tuple3<T0, T1, T2>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple3<T0, T1, T2>>(fieldIndexes, outType));
}
@@ -173,7 +173,7 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple4<T0, T1, T2, T3>> outType = (TypeInformation<Tuple4<T0, T1, T2, T3>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple4<T0, T1, T2, T3>>(fieldIndexes, outType));
}
@@ -206,14 +206,14 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple5<T0, T1, T2, T3, T4>> outType = (TypeInformation<Tuple5<T0, T1, T2, T3, T4>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple5<T0, T1, T2, T3, T4>>(fieldIndexes, outType));
}
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -227,7 +227,7 @@ public class StreamProjection<IN> {
* @param type5
* The class of field '5' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -243,14 +243,14 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple6<T0, T1, T2, T3, T4, T5>> outType = (TypeInformation<Tuple6<T0, T1, T2, T3, T4, T5>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple6<T0, T1, T2, T3, T4, T5>>(fieldIndexes, outType));
}
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -266,7 +266,7 @@ public class StreamProjection<IN> {
* @param type6
* The class of field '6' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -283,7 +283,7 @@ public class StreamProjection<IN> {
TypeInformation<Tuple7<T0, T1, T2, T3, T4, T5, T6>> outType = (TypeInformation<Tuple7<T0, T1, T2, T3, T4, T5, T6>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction("projection", null, inTypeInfo, outType,
+ .transform("projection", outType,
new ProjectInvokable<IN, Tuple7<T0, T1, T2, T3, T4, T5, T6>>(fieldIndexes,
outType));
}
@@ -291,7 +291,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -309,7 +309,7 @@ public class StreamProjection<IN> {
* @param type7
* The class of field '7' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -325,7 +325,7 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>> outType = (TypeInformation<Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple8<T0, T1, T2, T3, T4, T5, T6, T7>>(fieldIndexes,
outType));
}
@@ -333,7 +333,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -353,7 +353,7 @@ public class StreamProjection<IN> {
* @param type8
* The class of field '8' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -369,7 +369,7 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>> outType = (TypeInformation<Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple9<T0, T1, T2, T3, T4, T5, T6, T7, T8>>(fieldIndexes,
outType));
}
@@ -377,7 +377,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -399,7 +399,7 @@ public class StreamProjection<IN> {
* @param type9
* The class of field '9' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -415,7 +415,7 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>> outType = (TypeInformation<Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple10<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9>>(
fieldIndexes, outType));
}
@@ -423,7 +423,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -447,7 +447,7 @@ public class StreamProjection<IN> {
* @param type10
* The class of field '10' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -465,7 +465,7 @@ public class StreamProjection<IN> {
@SuppressWarnings("unchecked")
TypeInformation<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>> outType = (TypeInformation<Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
- return dataStream.addFunction("projection", null, inTypeInfo, outType,
+ return dataStream.transform("projection", outType,
new ProjectInvokable<IN, Tuple11<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>>(
fieldIndexes, outType));
}
@@ -473,7 +473,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -499,7 +499,7 @@ public class StreamProjection<IN> {
* @param type11
* The class of field '11' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -518,10 +518,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>> outType = (TypeInformation<Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple12<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11>>(
fieldIndexes, outType));
@@ -530,7 +528,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -558,7 +556,7 @@ public class StreamProjection<IN> {
* @param type12
* The class of field '12' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -577,10 +575,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>> outType = (TypeInformation<Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple13<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12>>(
fieldIndexes, outType));
@@ -589,7 +585,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -619,7 +615,7 @@ public class StreamProjection<IN> {
* @param type13
* The class of field '13' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -638,10 +634,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>> outType = (TypeInformation<Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple14<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13>>(
fieldIndexes, outType));
@@ -650,7 +644,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -682,7 +676,7 @@ public class StreamProjection<IN> {
* @param type14
* The class of field '14' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -702,10 +696,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>> outType = (TypeInformation<Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple15<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14>>(
fieldIndexes, outType));
@@ -714,7 +706,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -748,7 +740,7 @@ public class StreamProjection<IN> {
* @param type15
* The class of field '15' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -768,10 +760,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>> outType = (TypeInformation<Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple16<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15>>(
fieldIndexes, outType));
@@ -780,7 +770,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -816,7 +806,7 @@ public class StreamProjection<IN> {
* @param type16
* The class of field '16' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -836,10 +826,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>> outType = (TypeInformation<Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple17<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16>>(
fieldIndexes, outType));
@@ -848,7 +836,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -886,7 +874,7 @@ public class StreamProjection<IN> {
* @param type17
* The class of field '17' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -906,10 +894,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>> outType = (TypeInformation<Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple18<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17>>(
fieldIndexes, outType));
@@ -918,7 +904,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -958,7 +944,7 @@ public class StreamProjection<IN> {
* @param type18
* The class of field '18' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -979,10 +965,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>> outType = (TypeInformation<Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple19<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18>>(
fieldIndexes, outType));
@@ -991,7 +975,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -1033,7 +1017,7 @@ public class StreamProjection<IN> {
* @param type19
* The class of field '19' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -1054,10 +1038,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>> outType = (TypeInformation<Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple20<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19>>(
fieldIndexes, outType));
@@ -1066,7 +1048,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -1110,7 +1092,7 @@ public class StreamProjection<IN> {
* @param type20
* The class of field '20' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -1132,10 +1114,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>> outType = (TypeInformation<Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple21<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20>>(
fieldIndexes, outType));
@@ -1144,7 +1124,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -1190,7 +1170,7 @@ public class StreamProjection<IN> {
* @param type21
* The class of field '21' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -1212,10 +1192,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>> outType = (TypeInformation<Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple22<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21>>(
fieldIndexes, outType));
@@ -1224,7 +1202,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -1272,7 +1250,7 @@ public class StreamProjection<IN> {
* @param type22
* The class of field '22' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -1295,10 +1273,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>> outType = (TypeInformation<Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple23<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22>>(
fieldIndexes, outType));
@@ -1307,7 +1283,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -1357,7 +1333,7 @@ public class StreamProjection<IN> {
* @param type23
* The class of field '23' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -1380,10 +1356,8 @@ public class StreamProjection<IN> {
TypeInformation<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>> outType = (TypeInformation<Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
outType,
new ProjectInvokable<IN, Tuple24<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23>>(
fieldIndexes, outType));
@@ -1392,7 +1366,7 @@ public class StreamProjection<IN> {
/**
* Projects a {@link Tuple} {@link DataStream} to the previously selected
* fields. Requires the classes of the fields of the resulting Tuples.
- *
+ *
* @param type0
* The class of field '0' of the result Tuples.
* @param type1
@@ -1444,7 +1418,7 @@ public class StreamProjection<IN> {
* @param type24
* The class of field '24' of the result Tuples.
* @return The projected DataStream.
- *
+ *
* @see Tuple
* @see DataStream
*/
@@ -1467,10 +1441,9 @@ public class StreamProjection<IN> {
TypeInformation<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>> outType = (TypeInformation<Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>>) extractFieldTypes(
fieldIndexes, types, inTypeInfo);
return dataStream
- .addFunction(
+ .transform(
"projection",
- null,
- inTypeInfo,
+
outType,
new ProjectInvokable<IN, Tuple25<T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, T20, T21, T22, T23, T24>>(
fieldIndexes, outType));
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 788f28d..cb9cd04 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -50,7 +50,7 @@ import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
* into windows (predefined chunks). User defined function such as
* {@link #reduce(ReduceFunction)}, {@link #reduceGroup(GroupReduceFunction)} or
* aggregations can be applied to the windows.
- *
+ *
* @param <OUT>
* The output type of the {@link WindowedDataStream}
*/
@@ -124,8 +124,8 @@ public class WindowedDataStream<OUT> {
this.userEvicters = windowedDataStream.userEvicters;
this.allCentral = windowedDataStream.allCentral;
}
-
- public <F> F clean(F f){
+
+ public <F> F clean(F f) {
return dataStream.clean(f);
}
@@ -231,7 +231,7 @@ public class WindowedDataStream<OUT> {
* @return The transformed DataStream
*/
public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reduceFunction) {
- return dataStream.addFunction("NextGenWindowReduce", clean(reduceFunction), getType(), getType(),
+ return dataStream.transform("NextGenWindowReduce", getType(),
getReduceInvokable(reduceFunction));
}
@@ -255,7 +255,7 @@ public class WindowedDataStream<OUT> {
TypeInformation<R> outType = TypeExtractor
.getGroupReduceReturnTypes(reduceFunction, inType);
- return dataStream.addFunction("NextGenWindowReduce", clean(reduceFunction), inType, outType,
+ return dataStream.transform("NextGenWindowReduce", outType,
getReduceGroupInvokable(reduceFunction));
}
@@ -457,8 +457,8 @@ public class WindowedDataStream<OUT> {
private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregator) {
StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregator);
- SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.addFunction("windowReduce",
- clean(aggregator), getType(), getType(), invokable);
+ SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.transform("windowReduce",
+ getType(), invokable);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 59d56aa..e9581a1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -22,8 +22,6 @@ import java.io.Serializable;
import java.util.Collection;
import java.util.List;
-import org.apache.commons.lang3.SerializationException;
-import org.apache.commons.lang3.SerializationUtils;
import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.io.InputFormat;
@@ -264,14 +262,10 @@ public abstract class StreamExecutionEnvironment {
DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "elements",
outTypeInfo);
- try {
- SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
- jobGraphBuilder.addStreamVertex(returnStream.getId(),
- new SourceInvokable<OUT>(function), null, outTypeInfo, "source",
- SerializationUtils.serialize(function), 1);
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize elements");
- }
+ SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
+ jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
+ null, outTypeInfo, "source", 1);
+
return returnStream;
}
@@ -300,15 +294,8 @@ public abstract class StreamExecutionEnvironment {
DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "collection",
outTypeInfo);
- try {
- SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
-
- jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(
- new FromElementsFunction<OUT>(data)), null, outTypeInfo, "source",
- SerializationUtils.serialize(function), 1);
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize collection");
- }
+ jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(
+ new FromElementsFunction<OUT>(data)), null, outTypeInfo, "source", 1);
return returnStream;
}
@@ -317,7 +304,7 @@ public abstract class StreamExecutionEnvironment {
* Creates a new DataStream that contains the strings received infinitely
* from socket. Received strings are decoded by the system's default
* character set.
- *
+ *
* @param hostname
* The host name which a server socket bind.
* @param port
@@ -335,7 +322,7 @@ public abstract class StreamExecutionEnvironment {
* Creates a new DataStream that contains the strings received infinitely
* from socket. Received strings are decoded by the system's default
* character set, uses '\n' as delimiter.
- *
+ *
* @param hostname
* The host name which a server socket bind.
* @param port
@@ -378,13 +365,8 @@ public abstract class StreamExecutionEnvironment {
DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "source", outTypeInfo);
- try {
- //currently parallel sources are not directly supported
- jobGraphBuilder.addSourceVertex(returnStream.getId(), function, null, outTypeInfo,
- "source", SerializationUtils.serialize(function), 1);
- } catch (SerializationException e) {
- throw new RuntimeException("Cannot serialize SourceFunction");
- }
+ jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
+ null, outTypeInfo, "source", 1);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
index cb960dd..8afac75 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FromElementsFunction.java
@@ -1,45 +1,49 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.flink.util.Collector;
-
-public class FromElementsFunction<T> implements SourceFunction<T> {
- private static final long serialVersionUID = 1L;
-
- Iterable<T> iterable;
-
- public FromElementsFunction(T... elements) {
- this.iterable = Arrays.asList(elements);
- }
-
- public FromElementsFunction(Collection<T> elements) {
- this.iterable = elements;
- }
-
- @Override
- public void invoke(Collector<T> collector) throws Exception {
- for (T element : iterable) {
- collector.collect(element);
- }
- }
-
-}
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.flink.util.Collector;
+
+public class FromElementsFunction<T> implements SourceFunction<T> {
+ private static final long serialVersionUID = 1L;
+
+ Iterable<T> iterable;
+
+ public FromElementsFunction(T... elements) {
+ this.iterable = Arrays.asList(elements);
+ }
+
+ public FromElementsFunction(Collection<T> elements) {
+ this.iterable = elements;
+ }
+
+ public FromElementsFunction(Iterable<T> elements) {
+ this.iterable = elements;
+ }
+
+ @Override
+ public void invoke(Collector<T> collector) throws Exception {
+ for (T element : iterable) {
+ collector.collect(element);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
index 7504efd..d786d6b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
@@ -37,7 +37,7 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
protected int instanceID;
protected String name;
private static int numVertices = 0;
- protected Object function;
+
protected String functionName;
private InputHandler<IN> inputHandler;
@@ -72,7 +72,6 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
this.configuration = new StreamConfig(getTaskConfiguration());
this.name = configuration.getVertexName();
this.functionName = configuration.getFunctionName();
- this.function = configuration.getFunction(userClassLoader);
this.states = configuration.getOperatorStates(userClassLoader);
this.context = createRuntimeContext(name, this.states);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index e2d71e1..f55fc71 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -50,6 +50,12 @@ under the License.
<artifactId>flink-compiler</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.scala-lang</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
new file mode 100644
index 0000000..711ce7c
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.api.scala.ClosureCleaner
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import scala.reflect.ClassTag
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.streaming.api.invokable.operator.MapInvokable
+
+class DataStream[OUT](javaStream: JavaStream[OUT]) {
+
+ /* This code is originally from the Apache Spark project. */
+ /**
+ * Clean a closure to make it ready to serialized and send to tasks
+ * (removes unreferenced variables in $outer's, updates REPL variables)
+ * If <tt>checkSerializable</tt> is set, <tt>clean</tt> will also proactively
+ * check to see if <tt>f</tt> is serializable and throw a <tt>SparkException</tt>
+ * if not.
+ *
+ * @param f the closure to clean
+ * @param checkSerializable whether or not to immediately check <tt>f</tt> for serializability
+ * @throws <tt>SparkException<tt> if <tt>checkSerializable</tt> is set but <tt>f</tt> is not
+ * serializable
+ */
+ private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
+ ClosureCleaner.clean(f, checkSerializable)
+ f
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element of this DataStream.
+ */
+ def map[R: TypeInformation: ClassTag](fun: OUT => R): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+ val mapper = new MapFunction[OUT, R] {
+ val cleanFun = clean(fun)
+ def map(in: OUT): R = cleanFun(in)
+ }
+
+ new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[OUT, R](mapper)))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element of this DataStream.
+ */
+ def map[R: TypeInformation: ClassTag](mapper: MapFunction[OUT, R]): DataStream[R] = {
+ if (mapper == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+
+ new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[OUT, R](mapper)))
+ }
+
+ def print() = javaStream.print()
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/6b1fd156/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
new file mode 100644
index 0000000..df6c561
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+
+import org.apache.flink.streaming.api.environment.{ StreamExecutionEnvironment => JavaEnv }
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.commons.lang.Validate
+import scala.reflect.ClassTag
+import org.apache.flink.streaming.api.datastream.DataStreamSource
+import org.apache.flink.streaming.api.invokable.SourceInvokable
+import org.apache.flink.streaming.api.function.source.FromElementsFunction
+
+class StreamExecutionEnvironment(javaEnv: JavaEnv) {
+
+ /**
+ * Sets the degree of parallelism (DOP) for operations executed through this environment.
+ * Setting a DOP of x here will cause all operators (such as join, map, reduce) to run with
+ * x parallel instances. This value can be overridden by specific operations using
+ * [[DataStream.setParallelism]].
+ */
+ def setDegreeOfParallelism(degreeOfParallelism: Int): Unit = {
+ javaEnv.setDegreeOfParallelism(degreeOfParallelism)
+ }
+
+ /**
+ * Returns the default degree of parallelism for this execution environment. Note that this
+ * value can be overridden by individual operations using [[DataStream.setParallelism]]
+ */
+ def getDegreeOfParallelism = javaEnv.getDegreeOfParallelism
+
+ def generateSequence(from: Long, to: Long): DataStream[java.lang.Long] = new DataStream(javaEnv.generateSequence(from, to))
+
+ /**
+ * Creates a new data stream that contains the given elements. The elements must all be of the
+ * same type and must be serializable.
+ *
+ * * Note that this operation will result in a non-parallel data source, i.e. a data source with
+ * a degree of parallelism of one.
+ */
+ def fromElements[T: ClassTag: TypeInformation](data: T*): DataStream[T] = {
+ val typeInfo = implicitly[TypeInformation[T]]
+ fromCollection(data)(implicitly[ClassTag[T]], typeInfo)
+ }
+
+ /**
+ * Creates a DataStream from the given non-empty [[Seq]]. The elements need to be serializable
+ * because the framework may move the elements into the cluster if needed.
+ *
+ * Note that this operation will result in a non-parallel data source, i.e. a data source with
+ * a degree of parallelism of one.
+ */
+ def fromCollection[T: ClassTag: TypeInformation](
+ data: Seq[T]): DataStream[T] = {
+ Validate.notNull(data, "Data must not be null.")
+ val typeInfo = implicitly[TypeInformation[T]]
+ val returnStream = new DataStreamSource[T](javaEnv,
+ "elements", typeInfo);
+
+ javaEnv.getJobGraphBuilder.addStreamVertex(returnStream.getId(),
+ new SourceInvokable[T](new FromElementsFunction[T](scala.collection.JavaConversions.asJavaCollection(data))), null, typeInfo,
+ "source", 1);
+ new DataStream(returnStream)
+ }
+
+ def execute() = javaEnv.execute()
+
+}
+
+object StreamExecutionEnvironment {
+
+ /**
+ * Creates an execution environment that represents the context in which the program is
+ * currently executed. If the program is invoked standalone, this method returns a local
+ * execution environment. If the program is invoked from within the command line client
+ * to be submitted to a cluster, this method returns the execution environment of this cluster.
+ */
+ def getExecutionEnvironment: StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.getExecutionEnvironment)
+ }
+
+ /**
+ * Creates a local execution environment. The local execution environment will run the program in
+ * a multi-threaded fashion in the same JVM as the environment was created in. The default degree
+ * of parallelism of the local environment is the number of hardware contexts (CPU cores/threads).
+ */
+ def createLocalEnvironment(
+ degreeOfParallelism: Int = Runtime.getRuntime.availableProcessors()): StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.createLocalEnvironment(degreeOfParallelism))
+ }
+
+ /**
+ * Creates a remote execution environment. The remote environment sends (parts of) the program to
+ * a cluster for execution. Note that all file paths used in the program must be accessible from
+ * the cluster. The execution will use the cluster's default degree of parallelism, unless the
+ * parallelism is set explicitly via [[ExecutionEnvironment.setDegreeOfParallelism()]].
+ *
+ * @param host The host name or address of the master (JobManager),
+ * where the program should be executed.
+ * @param port The port of the master (JobManager), where the program should be executed.
+ * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
+ * program uses
+ * user-defined functions, user-defined input formats, or any libraries,
+ * those must be
+ * provided in the JAR files.
+ */
+ def createRemoteEnvironment(host: String, port: Int, jarFiles: String*): StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*))
+ }
+
+ /**
+ * Creates a remote execution environment. The remote environment sends (parts of) the program
+ * to a cluster for execution. Note that all file paths used in the program must be accessible
+ * from the cluster. The execution will use the specified degree of parallelism.
+ *
+ * @param host The host name or address of the master (JobManager),
+ * where the program should be executed.
+ * @param port The port of the master (JobManager), where the program should be executed.
+ * @param degreeOfParallelism The degree of parallelism to use during the execution.
+ * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
+ * program uses
+ * user-defined functions, user-defined input formats, or any libraries,
+ * those must be
+ * provided in the JAR files.
+ */
+ def createRemoteEnvironment(
+ host: String,
+ port: Int,
+ degreeOfParallelism: Int,
+ jarFiles: String*): StreamExecutionEnvironment = {
+ val javaEnv = JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*)
+ javaEnv.setDegreeOfParallelism(degreeOfParallelism)
+ new StreamExecutionEnvironment(javaEnv)
+ }
+}
\ No newline at end of file
[14/36] flink git commit: [scala] [streaming] WindowJoin scala
example added
Posted by mb...@apache.org.
[scala] [streaming] WindowJoin scala example added
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c6b90eed
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c6b90eed
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c6b90eed
Branch: refs/heads/release-0.8
Commit: c6b90eed5e2a3e1ce51bdca436a2bebc06a87ca7
Parents: 1413484
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Dec 21 13:10:34 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:59:57 2015 +0100
----------------------------------------------------------------------
.../scala/streaming/windowing/WindowJoin.scala | 72 ++++++++++++++++++++
.../streaming/StreamExecutionEnvironment.scala | 16 +++++
2 files changed, 88 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c6b90eed/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
new file mode 100644
index 0000000..eea76c1
--- /dev/null
+++ b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.examples.scala.streaming.windowing
+
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.function.source.SourceFunction
+import org.apache.flink.util.Collector
+import scala.util.Random
+
+object WindowJoin {
+
+ case class Name(id: Long, name: String)
+ case class Age(id: Long, age: Int)
+ case class Person(name: String, age: Long)
+
+ def main(args: Array[String]) {
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+
+ //Create streams for names and ages by mapping the inputs to the corresponding objects
+ val names = env.addSource(nameStream _).map(x => Name(x._1, x._2))
+ val ages = env.addSource(ageStream _).map(x => Age(x._1, x._2))
+
+ //Join the two input streams by id on the last second and create new Person objects
+ //containing both name and age
+ val joined =
+ names.join(ages).onWindow(1000)
+ .where("id").equalTo("id") { (n, a) => Person(n.name, a.age) }
+
+ joined print
+
+ env.execute("WindowJoin")
+ }
+
+ //Stream source for generating (id, name) pairs
+ def nameStream(out: Collector[(Long, String)]) = {
+ val names = Array("tom", "jerry", "alice", "bob", "john", "grace")
+
+ for (i <- 1 to 10000) {
+ if (i % 100 == 0) Thread.sleep(1000) else {
+ out.collect((i, names(Random.nextInt(names.length))))
+ }
+ }
+ }
+
+ //Stream source for generating (id, age) pairs
+ def ageStream(out: Collector[(Long, Int)]) = {
+ for (i <- 1 to 10000) {
+ if (i % 100 == 0) Thread.sleep(1000) else {
+ out.collect((i, Random.nextInt(90)))
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/c6b90eed/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index 55f7c6c..2489a64 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -161,9 +161,25 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*/
def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
Validate.notNull(function, "Function must not be null.")
+ ClosureCleaner.clean(function, true)
val typeInfo = implicitly[TypeInformation[T]]
new DataStream[T](javaEnv.addSource(function, typeInfo))
}
+
+ /**
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality.
+ *
+ */
+ def addSource[T: ClassTag: TypeInformation](function: Collector[T] => Unit): DataStream[T] = {
+ Validate.notNull(function, "Function must not be null.")
+ val sourceFunction = new SourceFunction[T] {
+ override def invoke(out: Collector[T]) {
+ function(out)
+ }
+ }
+ addSource(sourceFunction)
+ }
/**
* Triggers the program execution. The environment will execute all parts of
[16/36] flink git commit: [scala] [streaming] Added scala window
helpers + timestamp rework for lambda support
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala
new file mode 100644
index 0000000..b7d1546
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming.windowing
+
+import org.apache.flink.streaming.api.windowing.helper.{ Delta => JavaDelta }
+import org.apache.commons.lang.Validate
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction
+
+object Delta {
+
+ /**
+ * Creates a delta helper representing a delta trigger or eviction policy.
+ * </br></br> This policy calculates a delta between the data point which
+ * triggered last and the currently arrived data point. It triggers if the
+ * delta is higher than a specified threshold. </br></br> In case it gets
+ * used for eviction, this policy starts from the first element of the
+ * buffer and removes all elements from the buffer which have a higher delta
+ * then the threshold. As soon as there is an element with a lower delta,
+ * the eviction stops.
+ */
+ def of[T](threshold: Double, deltaFunction: (T, T) => Double, initVal: T): JavaDelta[T] = {
+ Validate.notNull(deltaFunction, "Delta function must not be null")
+ val df = new DeltaFunction[T] {
+ val cleanFun = clean(deltaFunction)
+ override def getDelta(first: T, second: T) = cleanFun(first, second)
+ }
+ JavaDelta.of(threshold, df, initVal)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala
new file mode 100644
index 0000000..62a47c2
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming.windowing
+
+import java.util.concurrent.TimeUnit
+import org.apache.flink.streaming.api.windowing.helper.{ Time => JavaTime }
+import org.apache.flink.api.scala.ClosureCleaner
+import org.apache.commons.net.ntp.TimeStamp
+import org.apache.flink.streaming.api.windowing.helper.Timestamp
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
+import org.apache.commons.lang.Validate
+
+object Time {
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using System time.
+ *
+ */
+ def of(windowSize: Long, timeUnit: TimeUnit): JavaTime[_] =
+ JavaTime.of(windowSize, timeUnit)
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using a user defined timestamp extractor.
+ *
+ */
+ def of[R](windowSize: Long, timestamp: R => Long, startTime: Long = 0): JavaTime[R] = {
+ Validate.notNull(timestamp, "Timestamp must not be null.")
+ val ts = new Timestamp[R] {
+ val fun = clean(timestamp, true)
+ override def getTimestamp(in: R) = fun(in)
+ }
+ JavaTime.of(windowSize, ts, startTime)
+ }
+
+}
[19/36] flink git commit: [scala] [streaming] added group windowing
example with different policy types
Posted by mb...@apache.org.
[scala] [streaming] added group windowing example with different policy types
[scala] [streaming] updated example with scala delta helper usage
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e1fa9b5d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e1fa9b5d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e1fa9b5d
Branch: refs/heads/release-0.8
Commit: e1fa9b5db33bf5167e1cd56de862f482dd10c614
Parents: c34f5a7
Author: senorcarbone <se...@gmail.com>
Authored: Fri Jan 2 11:15:49 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:06:38 2015 +0100
----------------------------------------------------------------------
.../windowing/TopSpeedWindowingExample.java | 121 +++++++++++++++++++
.../streaming/windowing/TopSpeedWindowing.scala | 93 ++++++++++++++
2 files changed, 214 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e1fa9b5d/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
new file mode 100644
index 0000000..bc3bba5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+import org.apache.flink.streaming.api.windowing.helper.Delta;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.util.Collector;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An example of grouped stream windowing where different eviction and trigger policies can be used.
+ * A source fetches events from cars every 1 sec containing their id, their current speed (kmh),
+ * overall elapsed distance (m) and a timestamp. The streaming
+ * example triggers the top speed of each car every x meters elapsed for the last y seconds.
+ */
+public class TopSpeedWindowingExample {
+
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ DataStream topSpeeds = env
+ .addSource(CarSource.create(numOfCars))
+ .groupBy(0)
+ .window(Time.of(evictionSec, TimeUnit.SECONDS))
+ .every(Delta.of(
+ new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() {
+ @Override
+ public double getDelta(Tuple4<Integer, Integer, Double, Long> oldDataPoint, Tuple4<Integer, Integer, Double, Long> newDataPoint) {
+ return newDataPoint.f2 - oldDataPoint.f2;
+ }
+ }
+ , new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l), triggerMeters))
+ .maxBy(1);
+
+ topSpeeds.print();
+ env.execute("CarTopSpeedWindowingExample");
+ }
+
+ private static class CarSource implements SourceFunction<Tuple4<Integer, Integer, Double, Long>> {
+ private Integer[] speeds;
+ private Double[] distances;
+
+ private Random rand = new Random();
+
+ private CarSource(int numOfCars) {
+ speeds = new Integer[numOfCars];
+ distances = new Double[numOfCars];
+ Arrays.fill(speeds, 50);
+ Arrays.fill(distances, 0d);
+ }
+
+ public static CarSource create(int cars) {
+ return new CarSource(cars);
+ }
+
+ @Override
+ public void invoke(Collector<Tuple4<Integer, Integer, Double, Long>> collector) throws Exception {
+
+ while (true) {
+ Thread.sleep(1000);
+ for (int carId = 0; carId < speeds.length; carId++) {
+ if (rand.nextBoolean())
+ speeds[carId] = Math.min(100, speeds[carId] + 5);
+ else
+ speeds[carId] = Math.max(0, speeds[carId] - 5);
+ distances[carId] += speeds[carId] / 3.6d;
+ collector.collect(new Tuple4<Integer, Integer, Double, Long>(carId, speeds[carId], distances[carId], System.currentTimeMillis()));
+ }
+ }
+ }
+ }
+
+ private static int numOfCars = 2;
+ private static int evictionSec = 10;
+ private static double triggerMeters = 50;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ if (args.length == 3) {
+ numOfCars = Integer.valueOf(args[0]);
+ evictionSec = Integer.valueOf(args[1]);
+ triggerMeters = Double.valueOf(args[2]);
+ } else {
+ System.err.println("Usage: TopSpeedWindowingExample <numCars> <evictSec> <triggerMeters>");
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e1fa9b5d/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
new file mode 100644
index 0000000..10ddfd8
--- /dev/null
+++ b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.examples.scala.streaming.windowing
+
+
+import java.util.concurrent.TimeUnit._
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment
+import org.apache.flink.api.scala.streaming.windowing.Delta
+import org.apache.flink.streaming.api.windowing.helper.Time
+import org.apache.flink.util.Collector
+
+import scala.math.{max, min}
+import scala.util.Random
+
+/**
+ * An example of grouped stream windowing where different eviction and trigger policies can be used.
+ * A source fetches events from cars every 1 sec containing their id, their current speed (kmh),
+ * overall elapsed distance (m) and a timestamp. The streaming
+ * example triggers the top speed of each car every x meters elapsed for the last y seconds.
+ */
+object TopSpeedWindowing {
+
+ case class CarSpeed(carId: Int, speed: Int, distance: Double, time: Long)
+
+ def main(args: Array[String]) {
+ if (!parseParameters(args)) {
+ return
+ }
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val cars = env.addSource(carSource _).groupBy("carId")
+ .window(Time.of(evictionSec, SECONDS))
+ .every(Delta.of[CarSpeed](triggerMeters, (oldSp,newSp) => newSp.distance-oldSp.distance, CarSpeed(0,0,0,0)))
+ .reduce((x, y) => if (x.speed > y.speed) x else y)
+
+ cars print
+
+ env.execute("TopSpeedWindowing")
+
+ }
+
+ def carSource(out: Collector[CarSpeed]) = {
+
+ val speeds = new Array[Int](numOfCars)
+ val distances = new Array[Double](numOfCars)
+
+ while (true) {
+ Thread sleep 1000
+ for (i <- 0 until speeds.length) {
+ speeds(i) = if (Random.nextBoolean) min(100, speeds(i) + 5) else max(0, speeds(i) - 5)
+ distances(i) += speeds(i) / 3.6d
+ out.collect(new CarSpeed(i, speeds(i), distances(i), System.currentTimeMillis))
+ }
+ }
+ }
+
+ def parseParameters(args: Array[String]): Boolean = {
+ if (args.length > 0) {
+ if (args.length == 3) {
+ numOfCars = args(0).toInt
+ evictionSec = args(1).toInt
+ triggerMeters = args(2).toDouble
+ }
+ else {
+ System.err.println("Usage: TopSpeedWindowing <numCars> <evictSec> <triggerMeters>")
+ false
+ }
+ }
+ true
+ }
+
+ var numOfCars = 2
+ var evictionSec = 10
+ var triggerMeters = 50d
+
+}
[24/36] flink git commit: [streaming] [scala] Restructured streaming
scala project and examples
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
deleted file mode 100644
index bcd586e..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.environment.{ StreamExecutionEnvironment => JavaEnv }
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.commons.lang.Validate
-import scala.reflect.ClassTag
-import org.apache.flink.streaming.api.datastream.DataStreamSource
-import org.apache.flink.streaming.api.invokable.SourceInvokable
-import org.apache.flink.streaming.api.function.source.FromElementsFunction
-import org.apache.flink.streaming.api.function.source.SourceFunction
-import scala.collection.JavaConversions._
-import org.apache.flink.util.Collector
-import org.apache.flink.api.scala.streaming.StreamingConversions._
-
-class StreamExecutionEnvironment(javaEnv: JavaEnv) {
-
- /**
- * Sets the degree of parallelism (DOP) for operations executed through this environment.
- * Setting a DOP of x here will cause all operators (such as join, map, reduce) to run with
- * x parallel instances. This value can be overridden by specific operations using
- * [[DataStream.setParallelism]].
- */
- def setDegreeOfParallelism(degreeOfParallelism: Int): Unit = {
- javaEnv.setDegreeOfParallelism(degreeOfParallelism)
- }
-
- /**
- * Returns the default degree of parallelism for this execution environment. Note that this
- * value can be overridden by individual operations using [[DataStream.setParallelism]]
- */
- def getDegreeOfParallelism = javaEnv.getDegreeOfParallelism
-
- /**
- * Sets the maximum time frequency (milliseconds) for the flushing of the
- * output buffers. By default the output buffers flush frequently to provide
- * low latency and to aid smooth developer experience. Setting the parameter
- * can result in three logical modes:
- *
- * <ul>
- * <li>
- * A positive integer triggers flushing periodically by that integer</li>
- * <li>
- * 0 triggers flushing after every record thus minimizing latency</li>
- * <li>
- * -1 triggers flushing only when the output buffer is full thus maximizing
- * throughput</li>
- * </ul>
- *
- */
- def setBufferTimeout(timeoutMillis: Long): StreamExecutionEnvironment = {
- javaEnv.setBufferTimeout(timeoutMillis)
- this
- }
-
- /**
- * Gets the default buffer timeout set for this environment
- */
- def getBufferTimout: Long = javaEnv.getBufferTimeout()
-
- /**
- * Creates a DataStream that represents the Strings produced by reading the
- * given file line wise. The file will be read with the system's default
- * character set.
- *
- */
- def readTextFile(filePath: String): DataStream[String] =
- javaEnv.readTextFile(filePath)
-
- /**
- * Creates a DataStream that represents the Strings produced by reading the
- * given file line wise multiple times(infinite). The file will be read with
- * the system's default character set. This functionality can be used for
- * testing a topology.
- *
- */
- def readTextStream(StreamPath: String): DataStream[String] =
- javaEnv.readTextStream(StreamPath)
-
- /**
- * Creates a new DataStream that contains the strings received infinitely
- * from socket. Received strings are decoded by the system's default
- * character set.
- *
- */
- def socketTextStream(hostname: String, port: Int, delimiter: Char): DataStream[String] =
- javaEnv.socketTextStream(hostname, port, delimiter)
-
- /**
- * Creates a new DataStream that contains the strings received infinitely
- * from socket. Received strings are decoded by the system's default
- * character set, uses '\n' as delimiter.
- *
- */
- def socketTextStream(hostname: String, port: Int): DataStream[String] =
- javaEnv.socketTextStream(hostname, port)
-
- /**
- * Creates a new DataStream that contains a sequence of numbers.
- *
- */
- def generateSequence(from: Long, to: Long): DataStream[Long] = {
- new DataStream[java.lang.Long](javaEnv.generateSequence(from, to)).
- asInstanceOf[DataStream[Long]]
- }
-
- /**
- * Creates a DataStream that contains the given elements. The elements must all be of the
- * same type and must be serializable.
- *
- * * Note that this operation will result in a non-parallel data source, i.e. a data source with
- * a degree of parallelism of one.
- */
- def fromElements[T: ClassTag: TypeInformation](data: T*): DataStream[T] = {
- val typeInfo = implicitly[TypeInformation[T]]
- fromCollection(data)(implicitly[ClassTag[T]], typeInfo)
- }
-
- /**
- * Creates a DataStream from the given non-empty [[Seq]]. The elements need to be serializable
- * because the framework may move the elements into the cluster if needed.
- *
- * Note that this operation will result in a non-parallel data source, i.e. a data source with
- * a degree of parallelism of one.
- */
- def fromCollection[T: ClassTag: TypeInformation](
- data: Seq[T]): DataStream[T] = {
- Validate.notNull(data, "Data must not be null.")
- val typeInfo = implicitly[TypeInformation[T]]
-
- val sourceFunction = new FromElementsFunction[T](scala.collection.JavaConversions
- .asJavaCollection(data))
-
- javaEnv.addSource(sourceFunction, typeInfo)
- }
-
- /**
- * Create a DataStream using a user defined source function for arbitrary
- * source functionality.
- *
- */
- def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
- Validate.notNull(function, "Function must not be null.")
- val cleanFun = StreamExecutionEnvironment.clean(function)
- val typeInfo = implicitly[TypeInformation[T]]
- javaEnv.addSource(cleanFun, typeInfo)
- }
-
- /**
- * Create a DataStream using a user defined source function for arbitrary
- * source functionality.
- *
- */
- def addSource[T: ClassTag: TypeInformation](function: Collector[T] => Unit): DataStream[T] = {
- Validate.notNull(function, "Function must not be null.")
- val sourceFunction = new SourceFunction[T] {
- val cleanFun = StreamExecutionEnvironment.clean(function)
- override def invoke(out: Collector[T]) {
- cleanFun(out)
- }
- }
- addSource(sourceFunction)
- }
-
- /**
- * Triggers the program execution. The environment will execute all parts of
- * the program that have resulted in a "sink" operation. Sink operations are
- * for example printing results or forwarding them to a message queue.
- * <p>
- * The program execution will be logged and displayed with a generated
- * default name.
- *
- */
- def execute() = javaEnv.execute()
-
- /**
- * Triggers the program execution. The environment will execute all parts of
- * the program that have resulted in a "sink" operation. Sink operations are
- * for example printing results or forwarding them to a message queue.
- * <p>
- * The program execution will be logged and displayed with the provided name
- *
- */
- def execute(jobName: String) = javaEnv.execute(jobName)
-
-}
-
-object StreamExecutionEnvironment {
-
- private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
- ClosureCleaner.clean(f, checkSerializable)
- f
- }
-
- /**
- * Creates an execution environment that represents the context in which the program is
- * currently executed. If the program is invoked standalone, this method returns a local
- * execution environment. If the program is invoked from within the command line client
- * to be submitted to a cluster, this method returns the execution environment of this cluster.
- */
- def getExecutionEnvironment: StreamExecutionEnvironment = {
- new StreamExecutionEnvironment(JavaEnv.getExecutionEnvironment)
- }
-
- /**
- * Creates a local execution environment. The local execution environment will run the program in
- * a multi-threaded fashion in the same JVM as the environment was created in. The default degree
- * of parallelism of the local environment is the number of hardware contexts (CPU cores/threads).
- */
- def createLocalEnvironment(
- degreeOfParallelism: Int = Runtime.getRuntime.availableProcessors()):
- StreamExecutionEnvironment = {
- new StreamExecutionEnvironment(JavaEnv.createLocalEnvironment(degreeOfParallelism))
- }
-
- /**
- * Creates a remote execution environment. The remote environment sends (parts of) the program to
- * a cluster for execution. Note that all file paths used in the program must be accessible from
- * the cluster. The execution will use the cluster's default degree of parallelism, unless the
- * parallelism is set explicitly via [[StreamExecutionEnvironment.setDegreeOfParallelism()]].
- *
- * @param host The host name or address of the master (JobManager),
- * where the program should be executed.
- * @param port The port of the master (JobManager), where the program should be executed.
- * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
- * program uses
- * user-defined functions, user-defined input formats, or any libraries,
- * those must be
- * provided in the JAR files.
- */
- def createRemoteEnvironment(host: String, port: Int, jarFiles: String*):
- StreamExecutionEnvironment = {
- new StreamExecutionEnvironment(JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*))
- }
-
- /**
- * Creates a remote execution environment. The remote environment sends (parts of) the program
- * to a cluster for execution. Note that all file paths used in the program must be accessible
- * from the cluster. The execution will use the specified degree of parallelism.
- *
- * @param host The host name or address of the master (JobManager),
- * where the program should be executed.
- * @param port The port of the master (JobManager), where the program should be executed.
- * @param degreeOfParallelism The degree of parallelism to use during the execution.
- * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
- * program uses
- * user-defined functions, user-defined input formats, or any libraries,
- * those must be
- * provided in the JAR files.
- */
- def createRemoteEnvironment(
- host: String,
- port: Int,
- degreeOfParallelism: Int,
- jarFiles: String*): StreamExecutionEnvironment = {
- val javaEnv = JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*)
- javaEnv.setDegreeOfParallelism(degreeOfParallelism)
- new StreamExecutionEnvironment(javaEnv)
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
deleted file mode 100644
index 8d8a0b0..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-
-import org.apache.flink.api.common.functions.JoinFunction
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.scala.ClosureCleaner
-import org.apache.flink.api.scala.typeutils.CaseClassSerializer
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.datastream.TemporalOperator
-import org.apache.flink.streaming.api.function.co.JoinWindowFunction
-import scala.reflect.ClassTag
-import org.apache.commons.lang.Validate
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
-import org.apache.flink.streaming.util.keys.KeySelectorUtil
-import org.apache.flink.api.java.operators.Keys
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
-import org.apache.flink.api.scala.streaming.StreamingConversions._
-
-class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
-TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
-
- override def createNextWindowOperator() = {
- new StreamJoinOperator.JoinWindow[I1, I2](this)
- }
-}
-
-object StreamJoinOperator {
-
- class JoinWindow[I1, I2](private[flink] op: StreamJoinOperator[I1, I2]) {
-
- private[flink] val type1 = op.input1.getType();
-
- /**
- * Continues a temporal Join transformation by defining
- * the fields in the first stream to be used as keys for the join.
- * The resulting incomplete join can be completed by JoinPredicate.equalTo()
- * to define the second key.
- */
- def where(fields: Int*) = {
- new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(fields.toArray,type1),type1))
- }
-
- /**
- * Continues a temporal Join transformation by defining
- * the fields in the first stream to be used as keys for the join.
- * The resulting incomplete join can be completed by JoinPredicate.equalTo()
- * to define the second key.
- */
- def where(firstField: String, otherFields: String*) =
- new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(firstField +: otherFields.toArray,type1),type1))
-
- /**
- * Continues a temporal Join transformation by defining
- * the keyselector function that will be used to extract keys from the first stream
- * for the join.
- * The resulting incomplete join can be completed by JoinPredicate.equalTo()
- * to define the second key.
- */
- def where[K: TypeInformation](fun: (I1) => K) = {
- val keyType = implicitly[TypeInformation[K]]
- val keyExtractor = new KeySelector[I1, K] {
- val cleanFun = op.input1.clean(fun)
- def getKey(in: I1) = cleanFun(in)
- }
- new JoinPredicate[I1, I2](op, keyExtractor)
- }
-
- }
-
- class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2],
- private[flink] val keys1: KeySelector[I1, _]) {
- private[flink] var keys2: KeySelector[I2, _] = null
- private[flink] val type2 = op.input2.getType();
-
- /**
- * Creates a temporal join transformation by defining the second join key.
- * The returned transformation wrapes each joined element pair in a tuple2:
- * (first, second)
- * To define a custom wrapping, use JoinedStream.apply(...)
- */
- def equalTo(fields: Int*): JoinedStream[I1, I2] = {
- finish(KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(fields.toArray,type2),type2))
- }
-
- /**
- * Creates a temporal join transformation by defining the second join key.
- * The returned transformation wrapes each joined element pair in a tuple2:
- * (first, second)
- * To define a custom wrapping, use JoinedStream.apply(...)
- */
- def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
- finish(KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(firstField +: otherFields.toArray,type2),type2))
-
- /**
- * Creates a temporal join transformation by defining the second join key.
- * The returned transformation wrapes each joined element pair in a tuple2:
- * (first, second)
- * To define a custom wrapping, use JoinedStream.apply(...)
- */
- def equalTo[K: TypeInformation](fun: (I2) => K): JoinedStream[I1, I2] = {
- val keyType = implicitly[TypeInformation[K]]
- val keyExtractor = new KeySelector[I2, K] {
- val cleanFun = op.input1.clean(fun)
- def getKey(in: I2) = cleanFun(in)
- }
- finish(keyExtractor)
- }
-
- private def finish(keys2: KeySelector[I2, _]): JoinedStream[I1, I2] = {
- this.keys2 = keys2
- new JoinedStream[I1, I2](this, createJoinOperator())
- }
-
- private def createJoinOperator(): JavaStream[(I1, I2)] = {
-
- val returnType = new CaseClassTypeInfo[(I1, I2)](
-
- classOf[(I1, I2)], Seq(op.input1.getType, op.input2.getType), Array("_1", "_2")) {
-
- override def createSerializer: TypeSerializer[(I1, I2)] = {
- val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
- for (i <- 0 until getArity) {
- fieldSerializers(i) = types(i).createSerializer
- }
-
- new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
- override def createInstance(fields: Array[AnyRef]) = {
- (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
- }
- }
- }
- }
-
- return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2))
- .addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
- returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
- }
- }
-
- class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends
- DataStream[(I1, I2)](javaStream) {
-
- private val op = jp.op
-
- /**
- * Sets a wrapper for the joined elements. For each joined pair, the result of the
- * udf call will be emitted.
- */
- def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
-
- val invokable = new CoWindowInvokable[I1, I2, R](
- clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
- op.timeStamp2)
-
- javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
- invokable)
-
- javaStream.setType(implicitly[TypeInformation[R]])
- }
- }
-
- private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2],
- joinFunction: (I1, I2) => R) = {
- Validate.notNull(joinFunction, "Join function must not be null.")
-
- val joinFun = new JoinFunction[I1, I2, R] {
-
- val cleanFun = jp.op.input1.clean(joinFunction)
-
- override def join(first: I1, second: I2): R = {
- cleanFun(first, second)
- }
- }
-
- new JoinWindowFunction[I1, I2, R](jp.keys1, jp.keys2, joinFun)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
deleted file mode 100644
index 9aefa04..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
-import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
-import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
-
-object StreamingConversions {
-
- implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
- new DataStream[R](javaStream)
-
- implicit def javaToScalaWindowedStream[R](javaWStream: JavaWStream[R]): WindowedDataStream[R] =
- new WindowedDataStream[R](javaWStream)
-
- implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
- new SplitDataStream[R](javaStream)
-
- implicit def javaToScalaConnectedStream[IN1, IN2](javaStream: JavaConStream[IN1, IN2]):
- ConnectedDataStream[IN1, IN2] = new ConnectedDataStream[IN1, IN2](javaStream)
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
deleted file mode 100644
index 2f9c792..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import scala.reflect.ClassTag
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.functions.MapFunction
-import org.apache.flink.streaming.api.invokable.operator.MapInvokable
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator
-import org.apache.flink.util.Collector
-import org.apache.flink.api.common.functions.FlatMapFunction
-import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
-import org.apache.flink.api.common.functions.ReduceFunction
-import org.apache.flink.api.common.functions.ReduceFunction
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
-import org.apache.flink.api.common.functions.GroupReduceFunction
-import org.apache.flink.streaming.api.invokable.StreamInvokable
-import scala.collection.JavaConversions._
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.streaming.api.function.aggregation.SumFunction
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
-import org.apache.flink.api.scala.streaming.StreamingConversions._
-
-class WindowedDataStream[T](javaStream: JavaWStream[T]) {
-
- /**
- * Defines the slide size (trigger frequency) for the windowed data stream.
- * This controls how often the user defined function will be triggered on
- * the window.
- */
- def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
- javaStream.every(windowingHelper: _*)
-
- /**
- * Groups the elements of the WindowedDataStream using the given
- * field positions. The window sizes (evictions) and slide sizes
- * (triggers) will be calculated on the whole stream (in a central fashion),
- * but the user defined functions will be applied on a per group basis.
- * </br></br> To get windows and triggers on a per group basis apply the
- * DataStream.window(...) operator on an already grouped data stream.
- *
- */
- def groupBy(fields: Int*): WindowedDataStream[T] = javaStream.groupBy(fields: _*)
-
- /**
- * Groups the elements of the WindowedDataStream using the given
- * field expressions. The window sizes (evictions) and slide sizes
- * (triggers) will be calculated on the whole stream (in a central fashion),
- * but the user defined functions will be applied on a per group basis.
- * </br></br> To get windows and triggers on a per group basis apply the
- * DataStream.window(...) operator on an already grouped data stream.
- *
- */
- def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
- javaStream.groupBy(firstField +: otherFields.toArray: _*)
-
- /**
- * Groups the elements of the WindowedDataStream using the given
- * KeySelector function. The window sizes (evictions) and slide sizes
- * (triggers) will be calculated on the whole stream (in a central fashion),
- * but the user defined functions will be applied on a per group basis.
- * </br></br> To get windows and triggers on a per group basis apply the
- * DataStream.window(...) operator on an already grouped data stream.
- *
- */
- def groupBy[K: TypeInformation](fun: T => K): WindowedDataStream[T] = {
-
- val keyExtractor = new KeySelector[T, K] {
- val cleanFun = clean(fun)
- def getKey(in: T) = cleanFun(in)
- }
- javaStream.groupBy(keyExtractor)
- }
-
- /**
- * Applies a reduce transformation on the windowed data stream by reducing
- * the current window at every trigger.
- *
- */
- def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
- if (reducer == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- javaStream.reduce(reducer)
- }
-
- /**
- * Applies a reduce transformation on the windowed data stream by reducing
- * the current window at every trigger.
- *
- */
- def reduce(fun: (T, T) => T): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- val reducer = new ReduceFunction[T] {
- val cleanFun = clean(fun)
- def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
- }
- reduce(reducer)
- }
-
- /**
- * Applies a reduceGroup transformation on the windowed data stream by reducing
- * the current window at every trigger. In contrast with the simple binary reduce operator,
- * groupReduce exposes the whole window through the Iterable interface.
- * </br>
- * </br>
- * Whenever possible try to use reduce instead of groupReduce for increased efficiency
- */
- def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
- DataStream[R] = {
- if (reducer == null) {
- throw new NullPointerException("GroupReduce function must not be null.")
- }
- javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]])
- }
-
- /**
- * Applies a reduceGroup transformation on the windowed data stream by reducing
- * the current window at every trigger. In contrast with the simple binary reduce operator,
- * groupReduce exposes the whole window through the Iterable interface.
- * </br>
- * </br>
- * Whenever possible try to use reduce instead of groupReduce for increased efficiency
- */
- def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
- DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("GroupReduce function must not be null.")
- }
- val reducer = new GroupReduceFunction[T, R] {
- val cleanFun = clean(fun)
- def reduce(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
- }
- reduceGroup(reducer)
- }
-
- /**
- * Applies an aggregation that that gives the maximum of the elements in the window at
- * the given position.
- *
- */
- def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
-
- /**
- * Applies an aggregation that that gives the minimum of the elements in the window at
- * the given position.
- *
- */
- def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
-
- /**
- * Applies an aggregation that sums the elements in the window at the given position.
- *
- */
- def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
-
- /**
- * Applies an aggregation that that gives the maximum element of the window by
- * the given position. When equality, returns the first.
- *
- */
- def maxBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MAXBY,
- position, first)
-
- /**
- * Applies an aggregation that that gives the minimum element of the window by
- * the given position. When equality, returns the first.
- *
- */
- def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MINBY,
- position, first)
-
- def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true):
- DataStream[T] = {
-
- val jStream = javaStream.asInstanceOf[JavaWStream[Product]]
- val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
-
- val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
-
- val reducer = aggregationType match {
- case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(
- outType.getTypeAt(position).getTypeClass()));
- case _ => new agg.ProductComparableAggregator(aggregationType, first)
- }
-
- new DataStream[Product](jStream.reduce(reducer)).asInstanceOf[DataStream[T]]
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala
deleted file mode 100644
index b7d1546..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Delta.scala
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming.windowing
-
-import org.apache.flink.streaming.api.windowing.helper.{ Delta => JavaDelta }
-import org.apache.commons.lang.Validate
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction
-
-object Delta {
-
- /**
- * Creates a delta helper representing a delta trigger or eviction policy.
- * </br></br> This policy calculates a delta between the data point which
- * triggered last and the currently arrived data point. It triggers if the
- * delta is higher than a specified threshold. </br></br> In case it gets
- * used for eviction, this policy starts from the first element of the
- * buffer and removes all elements from the buffer which have a higher delta
- * then the threshold. As soon as there is an element with a lower delta,
- * the eviction stops.
- */
- def of[T](threshold: Double, deltaFunction: (T, T) => Double, initVal: T): JavaDelta[T] = {
- Validate.notNull(deltaFunction, "Delta function must not be null")
- val df = new DeltaFunction[T] {
- val cleanFun = clean(deltaFunction)
- override def getDelta(first: T, second: T) = cleanFun(first, second)
- }
- JavaDelta.of(threshold, df, initVal)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala
deleted file mode 100644
index 62a47c2..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/windowing/Time.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming.windowing
-
-import java.util.concurrent.TimeUnit
-import org.apache.flink.streaming.api.windowing.helper.{ Time => JavaTime }
-import org.apache.flink.api.scala.ClosureCleaner
-import org.apache.commons.net.ntp.TimeStamp
-import org.apache.flink.streaming.api.windowing.helper.Timestamp
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
-import org.apache.commons.lang.Validate
-
-object Time {
-
- /**
- * Creates a helper representing a time trigger which triggers every given
- * length (slide size) or a time eviction which evicts all elements older
- * than length (window size) using System time.
- *
- */
- def of(windowSize: Long, timeUnit: TimeUnit): JavaTime[_] =
- JavaTime.of(windowSize, timeUnit)
-
- /**
- * Creates a helper representing a time trigger which triggers every given
- * length (slide size) or a time eviction which evicts all elements older
- * than length (window size) using a user defined timestamp extractor.
- *
- */
- def of[R](windowSize: Long, timestamp: R => Long, startTime: Long = 0): JavaTime[R] = {
- Validate.notNull(timestamp, "Timestamp must not be null.")
- val ts = new Timestamp[R] {
- val fun = clean(timestamp, true)
- override def getTimestamp(in: R) = fun(in)
- }
- JavaTime.of(windowSize, ts, startTime)
- }
-
-}
[17/36] flink git commit: [scala] [streaming] Added scala window
helpers + timestamp rework for lambda support
Posted by mb...@apache.org.
[scala] [streaming] Added scala window helpers + timestamp rework for lambda support
Conflicts:
flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1492e966
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1492e966
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1492e966
Branch: refs/heads/release-0.8
Commit: 1492e96670b6450e0c24ea31fe66d77c7d6a9c64
Parents: 5995248
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Jan 2 20:04:51 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:05:58 2015 +0100
----------------------------------------------------------------------
.../api/datastream/CoWindowDataStream.java | 20 +++---
.../api/datastream/ConnectedDataStream.java | 30 ++++----
.../api/datastream/TemporalOperator.java | 39 ++++++-----
.../co/CoGroupedWindowReduceInvokable.java | 4 +-
.../operator/co/CoWindowInvokable.java | 8 +--
.../operator/co/CoWindowReduceInvokable.java | 15 ++--
.../api/invokable/util/DefaultTimeStamp.java | 39 -----------
.../streaming/api/invokable/util/TimeStamp.java | 46 -------------
.../streaming/api/windowing/helper/Delta.java | 4 +-
.../api/windowing/helper/SystemTimestamp.java | 37 ++++++++++
.../streaming/api/windowing/helper/Time.java | 72 ++++++++++++++++----
.../api/windowing/helper/Timestamp.java | 39 +++++++++++
.../api/windowing/helper/TimestampWrapper.java | 44 ++++++++++++
.../windowing/policy/ActiveTriggerPolicy.java | 6 +-
.../windowing/policy/TimeEvictionPolicy.java | 31 +++++----
.../api/windowing/policy/TimeTriggerPolicy.java | 42 ++++++------
.../streaming/api/WindowCrossJoinTest.java | 30 ++++----
.../operator/CoGroupedWindowReduceTest.java | 5 +-
.../invokable/operator/CoWindowReduceTest.java | 5 +-
.../api/invokable/operator/CoWindowTest.java | 23 +++----
.../operator/GroupedWindowInvokableTest.java | 21 +++---
.../invokable/operator/WindowInvokableTest.java | 16 ++---
.../policy/TimeEvictionPolicyTest.java | 12 ++--
.../windowing/policy/TimeTriggerPolicyTest.java | 28 +++-----
.../examples/windowing/DeltaExtractExample.java | 6 +-
.../flink/api/scala/streaming/DataStream.scala | 19 +-----
.../scala/streaming/StreamCrossOperator.scala | 2 +-
.../streaming/StreamExecutionEnvironment.scala | 12 +++-
.../scala/streaming/StreamJoinOperator.scala | 1 +
.../scala/streaming/WindowedDataStream.scala | 6 +-
.../api/scala/streaming/windowing/Delta.scala | 47 +++++++++++++
.../api/scala/streaming/windowing/Time.scala | 55 +++++++++++++++
32 files changed, 454 insertions(+), 310 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java
index c8c634a..9129f9e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java
@@ -22,33 +22,33 @@ import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedWindowReduceInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowReduceInvokable;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
/**
* A {@link CoWindowDataStream} represents two data streams whose elements are
* batched together into sliding windows. Operation
* {@link #reduce(CoReduceFunction)} can be applied for each window.
- *
+ *
* @param <IN1>
* The type of the first input data stream
* @param <IN2>
* The type of the second input data stream
*/
public class CoWindowDataStream<IN1, IN2> extends CoBatchedDataStream<IN1, IN2> {
- TimeStamp<IN1> timeStamp1;
- TimeStamp<IN2> timeStamp2;
+ TimestampWrapper<IN1> timeStamp1;
+ TimestampWrapper<IN2> timeStamp2;
protected CoWindowDataStream(DataStream<IN1> dataStream1, DataStream<IN2> dataStream2,
long windowSize1, long windowSize2, long slideInterval1, long slideInterval2,
- TimeStamp<IN1> timeStamp1, TimeStamp<IN2> timeStamp2) {
+ TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
super(dataStream1, dataStream2, windowSize1, windowSize2, slideInterval1, slideInterval2);
this.timeStamp1 = timeStamp1;
this.timeStamp2 = timeStamp2;
}
protected CoWindowDataStream(ConnectedDataStream<IN1, IN2> coDataStream, long windowSize1,
- long windowSize2, long slideInterval1, long slideInterval2, TimeStamp<IN1> timeStamp1,
- TimeStamp<IN2> timeStamp2) {
+ long windowSize2, long slideInterval1, long slideInterval2,
+ TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
super(coDataStream, windowSize1, windowSize2, slideInterval1, slideInterval2);
this.timeStamp1 = timeStamp1;
this.timeStamp2 = timeStamp2;
@@ -96,9 +96,9 @@ public class CoWindowDataStream<IN1, IN2> extends CoBatchedDataStream<IN1, IN2>
CoReduceFunction<IN1, IN2, OUT> coReducer) {
CoWindowReduceInvokable<IN1, IN2, OUT> invokable;
if (isGrouped) {
- invokable = new CoGroupedWindowReduceInvokable<IN1, IN2, OUT>(clean(coReducer), batchSize1,
- batchSize2, slideSize1, slideSize2, keySelector1, keySelector2, timeStamp1,
- timeStamp2);
+ invokable = new CoGroupedWindowReduceInvokable<IN1, IN2, OUT>(clean(coReducer),
+ batchSize1, batchSize2, slideSize1, slideSize2, keySelector1, keySelector2,
+ timeStamp1, timeStamp2);
} else {
invokable = new CoWindowReduceInvokable<IN1, IN2, OUT>(clean(coReducer), batchSize1,
batchSize2, slideSize1, slideSize2, timeStamp1, timeStamp2);
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 39b6460..efd9531 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -36,8 +36,8 @@ import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
/**
* The ConnectedDataStream represents a stream for two different data types. It
@@ -305,8 +305,8 @@ public class ConnectedDataStream<IN1, IN2> {
* @return The transformed {@link ConnectedDataStream}
*/
public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2,
- long slideInterval1, long slideInterval2, TimeStamp<IN1> timeStamp1,
- TimeStamp<IN2> timeStamp2) {
+ long slideInterval1, long slideInterval2, TimestampWrapper<IN1> timeStamp1,
+ TimestampWrapper<IN2> timeStamp2) {
if (windowSize1 < 1 || windowSize2 < 1) {
throw new IllegalArgumentException("Window size must be positive");
}
@@ -338,10 +338,12 @@ public class ConnectedDataStream<IN1, IN2> {
* second input data stream are slid by after each transformation
* @return The transformed {@link ConnectedDataStream}
*/
+ @SuppressWarnings("unchecked")
public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2,
long slideInterval1, long slideInterval2) {
return window(windowSize1, windowSize2, slideInterval1, slideInterval2,
- new DefaultTimeStamp<IN1>(), new DefaultTimeStamp<IN2>());
+ (TimestampWrapper<IN1>) SystemTimestamp.getWrapper(),
+ (TimestampWrapper<IN2>) SystemTimestamp.getWrapper());
}
/**
@@ -365,7 +367,7 @@ public class ConnectedDataStream<IN1, IN2> {
* @return The transformed {@link ConnectedDataStream}
*/
public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2,
- TimeStamp<IN1> timeStamp1, TimeStamp<IN2> timeStamp2) {
+ TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
return window(windowSize1, windowSize2, windowSize1, windowSize2, timeStamp1, timeStamp2);
}
@@ -384,9 +386,11 @@ public class ConnectedDataStream<IN1, IN2> {
* milliseconds
* @return The transformed {@link ConnectedDataStream}
*/
+ @SuppressWarnings("unchecked")
public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2) {
return window(windowSize1, windowSize2, windowSize1, windowSize2,
- new DefaultTimeStamp<IN1>(), new DefaultTimeStamp<IN2>());
+ (TimestampWrapper<IN1>) SystemTimestamp.getWrapper(),
+ (TimestampWrapper<IN2>) SystemTimestamp.getWrapper());
}
/**
@@ -479,10 +483,12 @@ public class ConnectedDataStream<IN1, IN2> {
*
* @return The transformed {@link DataStream}.
*/
+ @SuppressWarnings("unchecked")
public <OUT> SingleOutputStreamOperator<OUT, ?> windowReduce(
CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize, long slideInterval) {
return windowReduce(coWindowFunction, windowSize, slideInterval,
- new DefaultTimeStamp<IN1>(), new DefaultTimeStamp<IN2>());
+ (TimestampWrapper<IN1>) SystemTimestamp.getWrapper(),
+ (TimestampWrapper<IN2>) SystemTimestamp.getWrapper());
}
/**
@@ -510,7 +516,7 @@ public class ConnectedDataStream<IN1, IN2> {
*/
public <OUT> SingleOutputStreamOperator<OUT, ?> windowReduce(
CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize, long slideInterval,
- TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
+ TimestampWrapper<IN1> timestamp1, TimestampWrapper<IN2> timestamp2) {
if (windowSize < 1) {
throw new IllegalArgumentException("Window size must be positive");
@@ -541,8 +547,8 @@ public class ConnectedDataStream<IN1, IN2> {
public <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
CoWindowFunction<IN1, IN2, OUT> coWindowFunction, TypeInformation<OUT> outTypeInfo,
- long windowSize, long slideInterval, TimeStamp<IN1> timestamp1,
- TimeStamp<IN2> timestamp2) {
+ long windowSize, long slideInterval, TimestampWrapper<IN1> timestamp1,
+ TimestampWrapper<IN2> timestamp2) {
if (windowSize < 1) {
throw new IllegalArgumentException("Window size must be positive");
@@ -550,7 +556,7 @@ public class ConnectedDataStream<IN1, IN2> {
if (slideInterval < 1) {
throw new IllegalArgumentException("Slide interval must be positive");
}
-
+
return addCoFunction("coWindowReduce", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
index cd8aabd..e5385f0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
@@ -18,8 +18,8 @@
package org.apache.flink.streaming.api.datastream;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
public abstract class TemporalOperator<I1, I2, OP> {
@@ -29,8 +29,8 @@ public abstract class TemporalOperator<I1, I2, OP> {
public long windowSize;
public long slideInterval;
- public TimeStamp<I1> timeStamp1;
- public TimeStamp<I2> timeStamp2;
+ public TimestampWrapper<I1> timeStamp1;
+ public TimestampWrapper<I2> timeStamp2;
public TemporalOperator(DataStream<I1> input1, DataStream<I2> input2) {
if (input1 == null || input2 == null) {
@@ -41,37 +41,37 @@ public abstract class TemporalOperator<I1, I2, OP> {
}
/**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be transformed.
*
* @param windowSize
* The size of the window in milliseconds.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
+ * @return An incomplete temporal transformation.
*/
public OP onWindow(long windowSize) {
return onWindow(windowSize, windowSize);
}
/**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be transformed.
*
* @param windowSize
* The size of the window in milliseconds.
* @param slideInterval
* The slide size of the window.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
+ * @return An incomplete temporal transformation.
*/
+ @SuppressWarnings("unchecked")
public OP onWindow(long windowSize, long slideInterval) {
- return onWindow(windowSize, slideInterval, new DefaultTimeStamp<I1>(),
- new DefaultTimeStamp<I2>());
+ return onWindow(windowSize, slideInterval,
+ (TimestampWrapper<I1>) SystemTimestamp.getWrapper(),
+ (TimestampWrapper<I2>) SystemTimestamp.getWrapper());
}
/**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be transformed.
*
* @param windowSize
* The size of the window in milliseconds.
@@ -83,11 +83,10 @@ public abstract class TemporalOperator<I1, I2, OP> {
* @param timeStamp2
* The timestamp used to extract time from the elements of the
* second data stream.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
+ * @return An incomplete temporal transformation.
*/
- public OP onWindow(long windowSize, long slideInterval, TimeStamp<I1> timeStamp1,
- TimeStamp<I2> timeStamp2) {
+ public OP onWindow(long windowSize, long slideInterval, TimestampWrapper<I1> timeStamp1,
+ TimestampWrapper<I2> timeStamp2) {
this.windowSize = windowSize;
this.slideInterval = slideInterval;
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java
index 4905566..736239f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
public class CoGroupedWindowReduceInvokable<IN1, IN2, OUT> extends
CoWindowReduceInvokable<IN1, IN2, OUT> {
@@ -38,7 +38,7 @@ public class CoGroupedWindowReduceInvokable<IN1, IN2, OUT> extends
public CoGroupedWindowReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer,
long windowSize1, long windowSize2, long slideInterval1, long slideInterval2,
KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2,
- TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
+ TimestampWrapper<IN1> timestamp1, TimestampWrapper<IN2> timestamp2) {
super(coReducer, windowSize1, windowSize2, slideInterval1, slideInterval2, timestamp1,
timestamp2);
this.keySelector1 = keySelector1;
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
index 7df5668..59552f4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.commons.math.util.MathUtils;
import org.apache.flink.streaming.api.function.co.CoWindowFunction;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.apache.flink.streaming.state.CircularFifoList;
public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
@@ -35,8 +35,8 @@ public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT>
protected long slideSize;
protected CircularFifoList<StreamRecord<IN1>> circularList1;
protected CircularFifoList<StreamRecord<IN2>> circularList2;
- protected TimeStamp<IN1> timeStamp1;
- protected TimeStamp<IN2> timeStamp2;
+ protected TimestampWrapper<IN1> timeStamp1;
+ protected TimestampWrapper<IN2> timeStamp2;
protected StreamWindow window;
@@ -44,7 +44,7 @@ public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT>
protected long nextRecordTime;
public CoWindowInvokable(CoWindowFunction<IN1, IN2, OUT> coWindowFunction, long windowSize,
- long slideInterval, TimeStamp<IN1> timeStamp1, TimeStamp<IN2> timeStamp2) {
+ long slideInterval, TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
super(coWindowFunction);
this.coWindowFunction = coWindowFunction;
this.windowSize = windowSize;
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java
index fa47761..0c8598f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java
@@ -19,8 +19,7 @@ package org.apache.flink.streaming.api.invokable.operator.co;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
public class CoWindowReduceInvokable<IN1, IN2, OUT> extends CoBatchReduceInvokable<IN1, IN2, OUT> {
private static final long serialVersionUID = 1L;
@@ -28,14 +27,14 @@ public class CoWindowReduceInvokable<IN1, IN2, OUT> extends CoBatchReduceInvokab
protected long startTime2;
protected long nextRecordTime1;
protected long nextRecordTime2;
- protected TimeStamp<IN1> timestamp1;
- protected TimeStamp<IN2> timestamp2;
+ protected TimestampWrapper<IN1> timestamp1;
+ protected TimestampWrapper<IN2> timestamp2;
protected StreamWindow<IN1> window1;
protected StreamWindow<IN2> window2;
public CoWindowReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer, long windowSize1,
- long windowSize2, long slideInterval1, long slideInterval2, TimeStamp<IN1> timestamp1,
- TimeStamp<IN2> timestamp2) {
+ long windowSize2, long slideInterval1, long slideInterval2,
+ TimestampWrapper<IN1> timestamp1, TimestampWrapper<IN2> timestamp2) {
super(coReducer, windowSize1, windowSize2, slideInterval1, slideInterval2);
this.timestamp1 = timestamp1;
this.timestamp2 = timestamp2;
@@ -51,10 +50,10 @@ public class CoWindowReduceInvokable<IN1, IN2, OUT> extends CoBatchReduceInvokab
this.window2 = new StreamWindow<IN2>(batchSize2, slideSize2);
this.batch1 = this.window1;
this.batch2 = this.window2;
- if (timestamp1 instanceof DefaultTimeStamp) {
+ if (timestamp1.isDefaultTimestamp()) {
(new TimeCheck1()).start();
}
- if (timestamp2 instanceof DefaultTimeStamp) {
+ if (timestamp2.isDefaultTimestamp()) {
(new TimeCheck2()).start();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/DefaultTimeStamp.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/DefaultTimeStamp.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/DefaultTimeStamp.java
deleted file mode 100644
index 2f22e8e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/DefaultTimeStamp.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.util;
-
-/**
- * Default timestamp function that uses the Java System.currentTimeMillis()
- * method to retrieve a timestamp.
- *
- * @param <T>
- * Type of the inputs of the reducing function.
- */
-public class DefaultTimeStamp<T> implements TimeStamp<T> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(T value) {
- return System.currentTimeMillis();
- }
-
- @Override
- public long getStartTime() {
- return System.currentTimeMillis();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/TimeStamp.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/TimeStamp.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/TimeStamp.java
deleted file mode 100644
index 86fa101..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/util/TimeStamp.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.util;
-
-import java.io.Serializable;
-
-/**
- * Interface for getting a timestamp from a custom value. Used in window
- * reduces. In order to work properly, the timestamps must be non-decreasing.
- *
- * @param <T>
- * Type of the value to create the timestamp from.
- */
-public interface TimeStamp<T> extends Serializable {
-
- /**
- * Values
- *
- * @param value
- * The value to create the timestamp from
- * @return The timestamp
- */
- public long getTimestamp(T value);
-
- /**
- * Function to define the starting time for reference
- *
- * @return The starting timestamp
- */
- public long getStartTime();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
index 9c8c5ca..5434a4e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
@@ -84,8 +84,8 @@ public class Delta<DATA> implements WindowingHelper<DATA> {
* The threshold used by the delta function.
* @return Helper representing a delta trigger or eviction policy
*/
- public static <DATA> Delta<DATA> of(DeltaFunction<DATA> deltaFunction, DATA initVal,
- double threshold) {
+ public static <DATA> Delta<DATA> of(double threshold, DeltaFunction<DATA> deltaFunction,
+ DATA initVal) {
return new Delta<DATA>(deltaFunction, initVal, threshold);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
new file mode 100644
index 0000000..8581ac5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/SystemTimestamp.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+/**
+ * {@link Timestamp} implementation to be used when system time is needed to
+ * determine windows
+ */
+public class SystemTimestamp<T> implements Timestamp<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(T value) {
+ return System.currentTimeMillis();
+ }
+
+ public static <R> TimestampWrapper<R> getWrapper() {
+ return new TimestampWrapper<R>(new SystemTimestamp<R>(), System.currentTimeMillis());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
index d987e32..9dc1c8c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -19,8 +19,6 @@ package org.apache.flink.streaming.api.windowing.helper;
import java.util.concurrent.TimeUnit;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
@@ -39,7 +37,7 @@ public class Time<DATA> implements WindowingHelper<DATA> {
private long length;
private TimeUnit granularity;
- private TimeStamp<DATA> timeStamp;
+ private TimestampWrapper<DATA> timestampWrapper;
private long delay;
/**
@@ -53,31 +51,52 @@ public class Time<DATA> implements WindowingHelper<DATA> {
* the smallest possible granularity is milliseconds. Any smaller
* time unit might cause an error at runtime due to conversion
* problems.
- * @param timeStamp
+ * @param timestamp
* The user defined timestamp that will be used to extract time
* information from the incoming elements
+ * @param startTime
+ * The startTime of the stream for computing the first window
*/
- private Time(long length, TimeUnit timeUnit, TimeStamp<DATA> timeStamp) {
+ private Time(long length, TimeUnit timeUnit, Timestamp<DATA> timestamp, long startTime) {
+ this(length, timeUnit, new TimestampWrapper<DATA>(timestamp, startTime));
+ }
+
+ /**
+ * Creates an helper representing a trigger which triggers every given
+ * length or an eviction which evicts all elements older than length.
+ *
+ * @param length
+ * The number of time units
+ * @param timeUnit
+ * The unit of time such as minute oder millisecond. Note that
+ * the smallest possible granularity is milliseconds. Any smaller
+ * time unit might cause an error at runtime due to conversion
+ * problems.
+ * @param timestampWrapper
+ * The user defined {@link TimestampWrapper} that will be used to
+ * extract time information from the incoming elements
+ */
+ private Time(long length, TimeUnit timeUnit, TimestampWrapper<DATA> timestampWrapper) {
this.length = length;
this.granularity = timeUnit;
- this.timeStamp = timeStamp;
+ this.timestampWrapper = timestampWrapper;
this.delay = 0;
}
@Override
public EvictionPolicy<DATA> toEvict() {
- return new TimeEvictionPolicy<DATA>(granularityInMillis(), timeStamp);
+ return new TimeEvictionPolicy<DATA>(granularityInMillis(), timestampWrapper);
}
@Override
public TriggerPolicy<DATA> toTrigger() {
- return new TimeTriggerPolicy<DATA>(granularityInMillis(), timeStamp, delay);
+ return new TimeTriggerPolicy<DATA>(granularityInMillis(), timestampWrapper, delay);
}
/**
* Creates a helper representing a time trigger which triggers every given
* length (slide size) or a time eviction which evicts all elements older
- * than length (window size).
+ * than length (window size) using System time.
*
* @param length
* The number of time units
@@ -88,8 +107,10 @@ public class Time<DATA> implements WindowingHelper<DATA> {
* problems.
* @return Helper representing the time based trigger and eviction policy
*/
+ @SuppressWarnings("unchecked")
public static <DATA> Time<DATA> of(long length, TimeUnit timeUnit) {
- return new Time<DATA>(length, timeUnit, new DefaultTimeStamp<DATA>());
+ return new Time<DATA>(length, timeUnit,
+ (TimestampWrapper<DATA>) SystemTimestamp.getWrapper());
}
/**
@@ -99,13 +120,32 @@ public class Time<DATA> implements WindowingHelper<DATA> {
*
* @param length
* The number of time units
- * @param timeStamp
+ * @param timestamp
+ * The user defined timestamp that will be used to extract time
+ * information from the incoming elements
+ * @param startTime
+ * The startTime used to compute the first window
+ * @return Helper representing the time based trigger and eviction policy
+ */
+ public static <DATA> Time<DATA> of(long length, Timestamp<DATA> timestamp, long startTime) {
+ return new Time<DATA>(length, null, timestamp, startTime);
+ }
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using a user defined timestamp extractor. By
+ * default the start time is set to 0.
+ *
+ * @param length
+ * The number of time units
+ * @param timestamp
* The user defined timestamp that will be used to extract time
* information from the incoming elements
* @return Helper representing the time based trigger and eviction policy
*/
- public static <DATA> Time<DATA> of(long length, TimeStamp<DATA> timeStamp) {
- return new Time<DATA>(length, TimeUnit.MILLISECONDS, timeStamp);
+ public static <DATA> Time<DATA> of(long length, Timestamp<DATA> timestamp) {
+ return of(length, timestamp, 0);
}
/**
@@ -121,6 +161,10 @@ public class Time<DATA> implements WindowingHelper<DATA> {
}
private long granularityInMillis() {
- return this.granularity.toMillis(this.length);
+ if (granularity != null) {
+ return this.granularity.toMillis(this.length);
+ } else {
+ return this.length;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
new file mode 100644
index 0000000..fea6020
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Timestamp.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import java.io.Serializable;
+
+/**
+ * Interface for getting a timestamp from a custom value. Used in window
+ * reduces. In order to work properly, the timestamps must be non-decreasing.
+ *
+ * @param <T>
+ * Type of the value to create the timestamp from.
+ */
+public interface Timestamp<T> extends Serializable {
+
+ /**
+ * Values
+ *
+ * @param value
+ * The value to create the timestamp from
+ * @return The timestamp
+ */
+ public long getTimestamp(T value);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
new file mode 100644
index 0000000..8c3a09d
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/TimestampWrapper.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import java.io.Serializable;
+
+public class TimestampWrapper<T> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private long startTime;
+ private Timestamp<T> timestamp;
+
+ public TimestampWrapper(Timestamp<T> timeStamp, long startTime) {
+ this.timestamp = timeStamp;
+ this.startTime = startTime;
+ }
+
+ public long getTimestamp(T in) {
+ return timestamp.getTimestamp(in);
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public boolean isDefaultTimestamp() {
+ return timestamp instanceof SystemTimestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
index a8a704d..414250c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
@@ -17,7 +17,7 @@
package org.apache.flink.streaming.api.windowing.policy;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
/**
* This interface extends the {@link TriggerPolicy} interface with functionality
@@ -28,7 +28,7 @@ import org.apache.flink.streaming.api.invokable.util.TimeStamp;
* first. It can return zero ore more fake data points which will be added
* before the the currently arrived real element gets processed. This allows to
* handle empty windows in time based windowing with an user defined
- * {@link TimeStamp}. Triggers are not called on fake datapoint. A fake
+ * {@link Timestamp}. Triggers are not called on fake datapoint. A fake
* datapoint is always considered as triggered.
*
* 2) An active trigger has a factory method for a runnable. This factory method
@@ -49,7 +49,7 @@ public interface ActiveTriggerPolicy<DATA> extends TriggerPolicy<DATA> {
* first. It can return zero ore more fake data points which will be added
* before the the currently arrived real element gets processed. This allows
* to handle empty windows in time based windowing with an user defined
- * {@link TimeStamp}. Triggers are not called on fake datapoints. A fake
+ * {@link Timestamp}. Triggers are not called on fake datapoints. A fake
* datapoint is always considered as triggered.
*
* @param datapoint
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
index aca1dee..16c30fc 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
@@ -19,14 +19,15 @@ package org.apache.flink.streaming.api.windowing.policy;
import java.util.LinkedList;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
/**
* This eviction policy evicts all elements which are older then a specified
- * time. The time is measured using a given {@link TimeStamp} implementation. A
+ * time. The time is measured using a given {@link Timestamp} implementation. A
* point in time is always represented as long. Therefore, the granularity can
* be set as long value as well.
- *
+ *
* @param <DATA>
* The type of the incoming data points which are processed by this
* policy.
@@ -40,12 +41,12 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
private static final long serialVersionUID = -1457476766124518220L;
private long granularity;
- private TimeStamp<DATA> timestamp;
+ private TimestampWrapper<DATA> timestampWrapper;
private LinkedList<Long> buffer = new LinkedList<Long>();
/**
* This eviction policy evicts all elements which are older than a specified
- * time. The time is measured using a given {@link TimeStamp}
+ * time. The time is measured using a given {@link Timestamp}
* implementation. A point in time is always represented as long. Therefore,
* the granularity can be set as long value as well. If this value is set to
* 2 the policy will evict all elements which are older as 2.
@@ -60,12 +61,12 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
* The granularity of the eviction. If this value is set to 2 the
* policy will evict all elements which are older as 2(if
* (time(X)<current time-granularity) evict X).
- * @param timestamp
- * The {@link TimeStamp} to measure the time with. This can be
- * either user defined of provided by the API.
+ * @param timestampWrapper
+ * The {@link TimestampWrapper} to measure the time with. This
+ * can be either user defined of provided by the API.
*/
- public TimeEvictionPolicy(long granularity, TimeStamp<DATA> timestamp) {
- this.timestamp = timestamp;
+ public TimeEvictionPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper) {
+ this.timestampWrapper = timestampWrapper;
this.granularity = granularity;
}
@@ -78,7 +79,7 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
try {
threshold = (Long) datapoint - granularity;
} catch (ClassCastException e) {
- threshold = timestamp.getTimestamp((DATA) datapoint) - granularity;
+ threshold = timestampWrapper.getTimestamp((DATA) datapoint) - granularity;
}
// return result
@@ -91,9 +92,9 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
checkForDeleted(bufferSize);
- //remember timestamp
- long time=timestamp.getTimestamp(datapoint);
-
+ // remember timestamp
+ long time = timestampWrapper.getTimestamp(datapoint);
+
// delete and count expired tuples
long threshold = time - granularity;
int counter = deleteAndCountExpired(threshold);
@@ -130,7 +131,7 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
@Override
public TimeEvictionPolicy<DATA> clone() {
- return new TimeEvictionPolicy<DATA>(granularity, timestamp);
+ return new TimeEvictionPolicy<DATA>(granularity, timestampWrapper);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
index 57bccf2..1e91b8e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -19,12 +19,12 @@ package org.apache.flink.streaming.api.windowing.policy;
import java.util.LinkedList;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
/**
* This trigger policy triggers with regard to the time. The is measured using a
- * given {@link TimeStamp} implementation. A point in time is always represented
+ * given {@link Timestamp} implementation. A point in time is always represented
* as long. Therefore, parameters such as granularity and delay can be set as
* long value as well.
*
@@ -42,12 +42,12 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
protected long startTime;
protected long granularity;
- protected TimeStamp<DATA> timestamp;
+ protected TimestampWrapper<DATA> timestampWrapper;
protected long delay;
/**
* This trigger policy triggers with regard to the time. The is measured
- * using a given {@link TimeStamp} implementation. A point in time is always
+ * using a given {@link Timestamp} implementation. A point in time is always
* represented as long. Therefore, parameters such as granularity can be set
* as long value as well. If this value for the granularity is set to 2 for
* example, the policy will trigger at every second point in time.
@@ -55,22 +55,22 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* @param granularity
* The granularity of the trigger. If this value is set to x the
* policy will trigger at every x-th time point
- * @param timestamp
- * The {@link TimeStamp} to measure the time with. This can be
- * either user defined of provided by the API.
+ * @param timestampWrapper
+ * The {@link TimestampWrapper} to measure the time with. This
+ * can be either user defined of provided by the API.
* @param timeWrapper
* This policy creates fake elements to not miss windows in case
* no element arrived within the duration of the window. This
* extractor should wrap a long into such an element of type
* DATA.
*/
- public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp) {
- this(granularity, timestamp, 0);
+ public TimeTriggerPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper) {
+ this(granularity, timestampWrapper, 0);
}
/**
* This is mostly the same as
- * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, TimeStamp)}. In addition
+ * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, Timestamp)}. In addition
* to granularity and timestamp a delay can be specified for the first
* trigger. If the start time given by the timestamp is x, the delay is y,
* and the granularity is z, the first trigger will happen at x+y+z.
@@ -78,9 +78,9 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* @param granularity
* The granularity of the trigger. If this value is set to 2 the
* policy will trigger at every second time point
- * @param timestamp
- * The {@link TimeStamp} to measure the time with. This can be
- * either user defined of provided by the API.
+ * @param timestampWrapper
+ * The {@link TimestampWrapper} to measure the time with. This
+ * can be either user defined of provided by the API.
* @param delay
* A delay for the first trigger. If the start time given by the
* timestamp is x, the delay is y, and the granularity is z, the
@@ -91,9 +91,9 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* extractor should wrap a long into such an element of type
* DATA.
*/
- public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp, long delay) {
- this.startTime = timestamp.getStartTime() + delay;
- this.timestamp = timestamp;
+ public TimeTriggerPolicy(long granularity, TimestampWrapper<DATA> timestampWrapper, long delay) {
+ this.startTime = timestampWrapper.getStartTime() + delay;
+ this.timestampWrapper = timestampWrapper;
this.granularity = granularity;
this.delay = delay;
}
@@ -107,7 +107,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
LinkedList<Object> fakeElements = new LinkedList<Object>();
// check if there is more then one window border missed
// use > here. In case >= would fit, the regular call will do the job.
- while (timestamp.getTimestamp(datapoint) >= startTime + granularity) {
+ while (timestampWrapper.getTimestamp(datapoint) >= startTime + granularity) {
startTime += granularity;
fakeElements.add(startTime - 1);
}
@@ -127,7 +127,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
*/
@Override
public Runnable createActiveTriggerRunnable(ActiveTriggerCallback callback) {
- if (this.timestamp instanceof DefaultTimeStamp) {
+ if (this.timestampWrapper.isDefaultTimestamp()) {
return new TimeCheck(callback);
} else {
return null;
@@ -177,7 +177,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
@Override
public synchronized boolean notifyTrigger(DATA datapoint) {
- long recordTime = timestamp.getTimestamp(datapoint);
+ long recordTime = timestampWrapper.getTimestamp(datapoint);
if (recordTime >= startTime + granularity) {
if (granularity != 0) {
startTime = recordTime - ((recordTime - startTime) % granularity);
@@ -190,7 +190,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
@Override
public TimeTriggerPolicy<DATA> clone() {
- return new TimeTriggerPolicy<DATA>(granularity, timestamp, delay);
+ return new TimeTriggerPolicy<DATA>(granularity, timestampWrapper, delay);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
index b71bb25..056d0a8 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -29,7 +29,7 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.junit.Test;
public class WindowCrossJoinTest implements Serializable {
@@ -96,12 +96,16 @@ public class WindowCrossJoinTest implements Serializable {
DataStream<Tuple2<Integer, String>> inStream1 = env.fromCollection(in1);
DataStream<Tuple1<Integer>> inStream2 = env.fromCollection(in2);
- inStream1.join(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
- .where(0).equalTo(0).addSink(new JoinResultSink());
+ inStream1
+ .join(inStream2)
+ .onWindow(1000, 1000, new MyTimestamp<Tuple2<Integer, String>>(),
+ new MyTimestamp<Tuple1<Integer>>()).where(0).equalTo(0)
+ .addSink(new JoinResultSink());
inStream1
.cross(inStream2)
- .onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
+ .onWindow(1000, 1000, new MyTimestamp<Tuple2<Integer, String>>(),
+ new MyTimestamp<Tuple1<Integer>>())
.with(new CrossFunction<Tuple2<Integer, String>, Tuple1<Integer>, Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>>() {
private static final long serialVersionUID = 1L;
@@ -119,25 +123,15 @@ public class WindowCrossJoinTest implements Serializable {
assertEquals(crossExpectedResults, crossResults);
}
- private static class MyTimestamp1 implements TimeStamp<Tuple2<Integer, String>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Tuple2<Integer, String> value) {
- return 101L;
+ private static class MyTimestamp<T> extends TimestampWrapper<T> {
+ public MyTimestamp() {
+ super(null, 0);
}
- @Override
- public long getStartTime() {
- return 100L;
- }
- }
-
- private static class MyTimestamp2 implements TimeStamp<Tuple1<Integer>> {
private static final long serialVersionUID = 1L;
@Override
- public long getTimestamp(Tuple1<Integer> value) {
+ public long getTimestamp(T value) {
return 101L;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
index e3f2a1b..508366c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
@@ -29,7 +29,7 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedWindowReduceInvokable;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.apache.flink.streaming.util.MockCoContext;
import org.junit.Test;
@@ -82,13 +82,14 @@ public class CoGroupedWindowReduceTest {
}
}
- public static final class MyTimeStamp<T> implements TimeStamp<T> {
+ public static final class MyTimeStamp<T> extends TimestampWrapper<T> {
private static final long serialVersionUID = 1L;
private Iterator<Long> timestamps;
private long start;
public MyTimeStamp(List<Long> timestamps) {
+ super(null, 0);
this.timestamps = timestamps.iterator();
this.start = timestamps.get(0);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java
index 90ad483..035a021 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java
@@ -27,7 +27,7 @@ import java.util.List;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowReduceInvokable;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.apache.flink.streaming.util.MockCoContext;
import org.junit.Test;
@@ -57,13 +57,14 @@ public class CoWindowReduceTest {
}
}
- public static final class MyTimeStamp<T> implements TimeStamp<T> {
+ public static final class MyTimeStamp<T> extends TimestampWrapper<T> {
private static final long serialVersionUID = 1L;
private Iterator<Long> timestamps;
private long start;
public MyTimeStamp(List<Long> timestamps) {
+ super(null, 0);
this.timestamps = timestamps.iterator();
this.start = timestamps.get(0);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
index c6d446a..4ab3492 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
@@ -27,7 +27,8 @@ import java.util.Set;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.function.co.CoWindowFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.apache.flink.streaming.util.MockCoContext;
import org.apache.flink.util.Collector;
import org.junit.Test;
@@ -80,7 +81,7 @@ public class CoWindowTest {
}
- private static final class MyTS1 implements TimeStamp<Integer> {
+ private static final class MyTS1 implements Timestamp<Integer> {
private static final long serialVersionUID = 1L;
@@ -89,14 +90,9 @@ public class CoWindowTest {
return value;
}
- @Override
- public long getStartTime() {
- return 1;
- }
-
}
- private static final class MyTS2 implements TimeStamp<Tuple2<Integer, Integer>> {
+ private static final class MyTS2 implements Timestamp<Tuple2<Integer, Integer>> {
private static final long serialVersionUID = 1L;
@@ -105,18 +101,14 @@ public class CoWindowTest {
return value.f0;
}
- @Override
- public long getStartTime() {
- return 1;
- }
-
}
@Test
public void coWindowGroupReduceTest2() throws Exception {
CoWindowInvokable<Integer, Integer, Integer> invokable1 = new CoWindowInvokable<Integer, Integer, Integer>(
- new MyCoGroup1(), 2, 1, new MyTS1(), new MyTS1());
+ new MyCoGroup1(), 2, 1, new TimestampWrapper<Integer>(new MyTS1(), 1),
+ new TimestampWrapper<Integer>(new MyTS1(), 1));
// Windowsize 2, slide 1
// 1,2|2,3|3,4|4,5
@@ -152,7 +144,8 @@ public class CoWindowTest {
assertEquals(expected1, actual1);
CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> invokable2 = new CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer>(
- new MyCoGroup2(), 2, 3, new MyTS2(), new MyTS2());
+ new MyCoGroup2(), 2, 3, new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(),
+ 1), new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(), 1));
// WindowSize 2, slide 3
// 1,2|4,5|7,8|
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
index d97cadc..f38d5c1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
@@ -28,7 +28,8 @@ import java.util.List;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
@@ -360,20 +361,18 @@ public class GroupedWindowInvokableTest {
expected.add(new Tuple2<Integer, String>(32, "b"));
expected.add(new Tuple2<Integer, String>(32, "c"));
- TimeStamp<Tuple2<Integer, String>> myTimeStamp = new TimeStamp<Tuple2<Integer, String>>() {
+ Timestamp<Tuple2<Integer, String>> myTimeStamp = new Timestamp<Tuple2<Integer, String>>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Tuple2<Integer, String> value) {
return value.f0;
}
-
- @Override
- public long getStartTime() {
- return 1;
- }
};
+ TimestampWrapper<Tuple2<Integer, String>> myTimeStampWrapper = new TimestampWrapper<Tuple2<Integer, String>>(
+ myTimeStamp, 1);
+
ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
private static final long serialVersionUID = 1L;
@@ -387,11 +386,11 @@ public class GroupedWindowInvokableTest {
LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
// Trigger every 2 time units but delay the first trigger by 2 (First
// trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L));
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
// Always delete all elements older then 4
- evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStamp));
+ evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
@@ -409,10 +408,10 @@ public class GroupedWindowInvokableTest {
// repeat the test with central eviction. The result should be the same.
triggers.clear();
- triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L));
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
evictions.clear();
LinkedList<EvictionPolicy<Tuple2<Integer, String>>> centralEvictions = new LinkedList<EvictionPolicy<Tuple2<Integer, String>>>();
- centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStamp));
+ centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
myReduceFunction, keySelector, distributedTriggers, evictions, triggers,
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
index 421a999..83b4596 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
@@ -24,7 +24,8 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
@@ -62,18 +63,13 @@ public class WindowInvokableTest {
expected.add(10);
expected.add(32);
- TimeStamp<Integer> myTimeStamp = new TimeStamp<Integer>() {
+ Timestamp<Integer> myTimeStamp = new Timestamp<Integer>() {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(Integer value) {
return value;
}
-
- @Override
- public long getStartTime() {
- return 1;
- }
};
ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
@@ -88,11 +84,13 @@ public class WindowInvokableTest {
LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
// Trigger every 2 time units but delay the first trigger by 2 (First
// trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Integer>(2L, myTimeStamp, 2L));
+ triggers.add(new TimeTriggerPolicy<Integer>(2L, new TimestampWrapper<Integer>(myTimeStamp,
+ 1), 2L));
LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
// Always delete all elements older then 4
- evictions.add(new TimeEvictionPolicy<Integer>(4L, myTimeStamp));
+ evictions.add(new TimeEvictionPolicy<Integer>(4L, new TimestampWrapper<Integer>(
+ myTimeStamp, 1)));
WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
myReduceFunction, triggers, evictions);
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
index 82c8841..b5d502b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
@@ -19,7 +19,8 @@ package org.apache.flink.streaming.api.windowing.policy;
import java.util.LinkedList;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -35,25 +36,20 @@ public class TimeEvictionPolicyTest {
// create a timestamp
@SuppressWarnings("serial")
- TimeStamp<Integer> timeStamp = new TimeStamp<Integer>() {
+ Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
@Override
public long getTimestamp(Integer value) {
return value;
}
- @Override
- public long getStartTime() {
- return 0;
- }
-
};
// test different granularity
for (long granularity = 0; granularity < 40; granularity++) {
// create policy
TimeEvictionPolicy<Integer> policy = new TimeEvictionPolicy<Integer>(granularity,
- timeStamp);
+ new TimestampWrapper<Integer>(timeStamp, 0));
// The trigger status should not effect the policy. Therefore, it's
// value is changed after each usage.
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
index 9c77a55..2bdbd96 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
@@ -21,7 +21,8 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
import org.junit.Test;
public class TimeTriggerPolicyTest {
@@ -33,29 +34,24 @@ public class TimeTriggerPolicyTest {
// create a timestamp
@SuppressWarnings("serial")
- TimeStamp<Integer> timeStamp = new TimeStamp<Integer>() {
+ Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
@Override
public long getTimestamp(Integer value) {
return value;
}
- @Override
- public long getStartTime() {
- return 0;
- }
-
};
// test different granularity
for (long granularity = 0; granularity < 31; granularity++) {
// create policy
- TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity, timeStamp);
+
+ TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity,
+ new TimestampWrapper<Integer>(timeStamp, 0));
// remember window border
- // Remark: This might NOT work in case the timeStamp uses
- // System.getCurrentTimeMillis to determine the start time.
- long currentTime = timeStamp.getStartTime();
+ long currentTime = 0;
// test by adding values
for (int i = 0; i < times.length; i++) {
@@ -85,22 +81,18 @@ public class TimeTriggerPolicyTest {
// create a timestamp
@SuppressWarnings("serial")
- TimeStamp<Integer> timeStamp = new TimeStamp<Integer>() {
+ Timestamp<Integer> timeStamp = new Timestamp<Integer>() {
@Override
public long getTimestamp(Integer value) {
return value;
}
- @Override
- public long getStartTime() {
- return 0;
- }
-
};
// create policy
- TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5, timeStamp);
+ TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5,
+ new TimestampWrapper<Integer>(timeStamp, 0));
// expected result
Long[][] result = { {}, {}, { 4L, 9L, 14L, 19L }, { 24L } };
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
index 1013e6f..d6a9ac0 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
@@ -49,10 +49,8 @@ public class DeltaExtractExample {
@SuppressWarnings({ "unchecked", "rawtypes" })
DataStream dstream = env
.addSource(new CountingSource())
- .window(Delta.of(new EuclideanDistance(new FieldsFromTuple(0, 1)), new Tuple3(0d,
- 0d, "foo"), 1.2))
- .every(Count.of(2))
- .reduce(new ConcatStrings());
+ .window(Delta.of(1.2, new EuclideanDistance(new FieldsFromTuple(0, 1)), new Tuple3(
+ 0d, 0d, "foo"))).every(Count.of(2)).reduce(new ConcatStrings());
// emit result
if (fileOutput) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 38ad384..dc1e5b3 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -18,6 +18,7 @@
package org.apache.flink.api.scala.streaming
import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
import org.apache.flink.api.common.typeinfo.TypeInformation
import scala.reflect.ClassTag
@@ -53,24 +54,6 @@ import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
class DataStream[T](javaStream: JavaStream[T]) {
- /* This code is originally from the Apache Spark project. */
- /**
- * Clean a closure to make it ready to serialized and send to tasks
- * (removes unreferenced variables in $outer's, updates REPL variables)
- * If <tt>checkSerializable</tt> is set, <tt>clean</tt> will also proactively
- * check to see if <tt>f</tt> is serializable and throw a <tt>SparkException</tt>
- * if not.
- *
- * @param f the closure to clean
- * @param checkSerializable whether or not to immediately check <tt>f</tt> for serializability
- * @throws <tt>SparkException<tt> if <tt>checkSerializable</tt> is set but <tt>f</tt> is not
- * serializable
- */
- private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
- ClosureCleaner.clean(f, checkSerializable)
- f
- }
-
/**
* Gets the underlying java DataStream object.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
index 72052b9..e9010c8 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
@@ -32,7 +32,7 @@ import org.apache.flink.streaming.api.function.co.CrossWindowFunction
import org.apache.flink.api.common.functions.CrossFunction
import org.apache.flink.api.scala.typeutils.CaseClassSerializer
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index 2489a64..a7a471f 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -161,9 +161,9 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*/
def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
Validate.notNull(function, "Function must not be null.")
- ClosureCleaner.clean(function, true)
+ val cleanFun = StreamExecutionEnvironment.clean(function)
val typeInfo = implicitly[TypeInformation[T]]
- new DataStream[T](javaEnv.addSource(function, typeInfo))
+ new DataStream[T](javaEnv.addSource(cleanFun, typeInfo))
}
/**
@@ -174,8 +174,9 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
def addSource[T: ClassTag: TypeInformation](function: Collector[T] => Unit): DataStream[T] = {
Validate.notNull(function, "Function must not be null.")
val sourceFunction = new SourceFunction[T] {
+ val cleanFun = StreamExecutionEnvironment.clean(function)
override def invoke(out: Collector[T]) {
- function(out)
+ cleanFun(out)
}
}
addSource(sourceFunction)
@@ -205,6 +206,11 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
}
object StreamExecutionEnvironment {
+
+ private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
+ ClosureCleaner.clean(f, checkSerializable)
+ f
+ }
/**
* Creates an execution environment that represents the context in which the program is
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
index f47d79e..4ed5082 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
@@ -33,6 +33,7 @@ import org.apache.commons.lang.Validate
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.util.keys.KeySelectorUtil
import org.apache.flink.api.java.operators.Keys
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1492e966/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
index 5346c4c..e33368c 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -40,14 +40,10 @@ import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.A
import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
import org.apache.flink.streaming.api.function.aggregation.SumFunction
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
class WindowedDataStream[T](javaStream: JavaWStream[T]) {
- private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
- ClosureCleaner.clean(f, checkSerializable)
- f
- }
-
/**
* Defines the slide size (trigger frequency) for the windowed data stream.
* This controls how often the user defined function will be triggered on
[10/36] flink git commit: [scala] [streaming] Temporal cross operator
added
Posted by mb...@apache.org.
[scala] [streaming] Temporal cross operator added
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/40a3b6bc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/40a3b6bc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/40a3b6bc
Branch: refs/heads/release-0.8
Commit: 40a3b6bc35ceff115a549f6094602d64b6eb69e9
Parents: 739fdf0
Author: Gyula Fora <gy...@apache.org>
Authored: Sat Dec 20 19:34:55 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:57:08 2015 +0100
----------------------------------------------------------------------
.../streaming/api/datastream/DataStream.java | 17 ++-
.../api/datastream/StreamCrossOperator.java | 59 ++++-------
.../flink/api/scala/streaming/DataStream.scala | 25 +++++
.../api/scala/streaming/FieldsKeySelector.scala | 18 ++++
.../scala/streaming/StreamCrossOperator.scala | 103 +++++++++++++++++++
.../scala/streaming/StreamJoinOperator.scala | 16 ++-
6 files changed, 183 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/40a3b6bc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 2a0b673..c4e3368 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -547,19 +547,16 @@ public class DataStream<OUT> {
* <p>
* This method returns a {@link StreamCrossOperator} on which the
* {@link StreamCrossOperator#onWindow} should be called to define the
- * window, and then call
- * {@link StreamCrossOperator.CrossWindow#with(org.apache.flink.api.common.functions.CrossFunction)}
- * to define a {@link org.apache.flink.api.common.functions.CrossFunction}
- * which is called for each pair of crossed elements. The CrossFunction
- * returns a exactly one element for each pair of input elements.
+ * window.
+ * <p>
+ * Call {@link StreamCrossOperator.CrossWindow#with(CrossFunction)} to
+ * define a custom cross function.
*
* @param dataStreamToCross
* The other DataStream with which this DataStream is crossed.
* @return A {@link StreamCrossOperator} to continue the definition of the
- * Join transformation.
+ * cross transformation.
*
- * @see org.apache.flink.api.common.functions.CrossFunction
- * @see DataStream
*/
public <IN2> StreamCrossOperator<OUT, IN2> cross(DataStream<IN2> dataStreamToCross) {
return new StreamCrossOperator<OUT, IN2>(this, dataStreamToCross);
@@ -574,7 +571,9 @@ public class DataStream<OUT> {
* {@link StreamJoinOperator#onWindow} should be called to define the
* window, and then the {@link StreamJoinOperator.JoinWindow#where} and
* {@link StreamJoinOperator.JoinPredicate#equalTo} can be used to define
- * the join keys.
+ * the join keys.</p> The user can also use the
+ * {@link StreamJoinOperator.JoinedStream#with(JoinFunction)} to apply
+ * custom join function.
*
* @param other
* The other DataStream with which this DataStream is joined.
http://git-wip-us.apache.org/repos/asf/flink/blob/40a3b6bc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
index c6cba63..2dd7bc0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
@@ -20,12 +20,12 @@ package org.apache.flink.streaming.api.datastream;
import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.ClosureCleaner;
-import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.operators.CrossOperator;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
public class StreamCrossOperator<I1, I2> extends
TemporalOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
@@ -36,23 +36,24 @@ public class StreamCrossOperator<I1, I2> extends
@Override
protected CrossWindow<I1, I2> createNextWindowOperator() {
- return new CrossWindow<I1, I2>(this);
+
+ CrossWindowFunction<I1, I2, Tuple2<I1, I2>> crossWindowFunction = new CrossWindowFunction<I1, I2, Tuple2<I1, I2>>(
+ input1.clean(new CrossOperator.DefaultCrossFunction<I1, I2>()));
+
+ return new CrossWindow<I1, I2>(this, input1.connect(input2).addGeneralWindowCombine(
+ crossWindowFunction,
+ new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType()), windowSize,
+ slideInterval, timeStamp1, timeStamp2));
}
- public static class CrossWindow<I1, I2> {
+ public static class CrossWindow<I1, I2> extends
+ SingleOutputStreamOperator<Tuple2<I1, I2>, CrossWindow<I1, I2>> {
private StreamCrossOperator<I1, I2> op;
- public CrossWindow(StreamCrossOperator<I1, I2> operator) {
- this.op = operator;
- }
-
- public <F> F clean(F f) {
- if (op.input1.getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
- ClosureCleaner.clean(f, true);
- }
- ClosureCleaner.ensureSerializable(f);
- return f;
+ public CrossWindow(StreamCrossOperator<I1, I2> op, DataStream<Tuple2<I1, I2>> ds) {
+ super(ds);
+ this.op = op;
}
/**
@@ -63,36 +64,20 @@ public class StreamCrossOperator<I1, I2> extends
* @param function
* The CrossFunction that is called for each pair of crossed
* elements.
- * @return A CrossOperator that represents the crossed result DataStream
+ * @return The crossed data streams
*
- * @see CrossFunction
- * @see DataSet
*/
public <R> SingleOutputStreamOperator<R, ?> with(CrossFunction<I1, I2, R> function) {
- return createCrossOperator(function);
- }
-
- /**
- * Finalizes a temporal Cross transformation by emitting all pairs in a
- * new Tuple2.
- *
- * @return A CrossOperator that represents the crossed result DataStream
- */
- public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> withDefault() {
- return createCrossOperator(new CrossOperator.DefaultCrossFunction<I1, I2>());
- }
-
- protected <R> SingleOutputStreamOperator<R, ?> createCrossOperator(
- CrossFunction<I1, I2, R> function) {
-
TypeInformation<R> outTypeInfo = TypeExtractor.getCrossReturnTypes(function,
op.input1.getType(), op.input2.getType());
- CrossWindowFunction<I1, I2, R> crossWindowFunction = new CrossWindowFunction<I1, I2, R>(
- clean(function));
+ CoWindowInvokable<I1, I2, R> invokable = new CoWindowInvokable<I1, I2, R>(
+ new CrossWindowFunction<I1, I2, R>(clean(function)), op.windowSize,
+ op.slideInterval, op.timeStamp1, op.timeStamp2);
+
+ jobGraphBuilder.setInvokable(id, invokable);
- return op.input1.connect(op.input2).addGeneralWindowCombine(crossWindowFunction,
- outTypeInfo, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2);
+ return setType(outTypeInfo);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/40a3b6bc/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 871fede..42ec709 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -458,9 +458,34 @@ class DataStream[T](javaStream: JavaStream[T]) {
split(selector)
}
+ /**
+ * Initiates a temporal Join transformation that joins the elements of two
+ * data streams on key equality over a specified time window.
+ *
+ * This method returns a StreamJoinOperator on which the
+ * .onWindow(..) should be called to define the
+ * window, and then the .where(..) and .equalTo(..) methods can be used to defin
+ * the join keys.</p> The user can also use the apply method of the returned JoinedStream
+ * to use custom join function.
+ *
+ */
def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] = new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
/**
+ * Initiates a temporal cross transformation that builds all pair
+ * combinations of elements of both DataStreams, i.e., it builds a Cartesian
+ * product.
+ *
+ * This method returns a StreamJoinOperator on which the
+ * .onWindow(..) should be called to define the
+ * window, and then the .where(..) and .equalTo(..) methods can be used to defin
+ * the join keys.</p> The user can also use the apply method of the returned JoinedStream
+ * to use custom join function.
+ *
+ */
+ def cross[R](stream: DataStream[R]): StreamCrossOperator[T, R] = new StreamCrossOperator[T, R](javaStream, stream.getJavaStream)
+
+ /**
* Writes a DataStream to the standard output stream (stdout). For each
* element of the DataStream the result of .toString is
* written.
http://git-wip-us.apache.org/repos/asf/flink/blob/40a3b6bc/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
index 4223512..d7c9f96 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.flink.api.scala.streaming
import org.apache.flink.streaming.util.keys.{ FieldsKeySelector => JavaSelector }
http://git-wip-us.apache.org/repos/asf/flink/blob/40a3b6bc/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
new file mode 100644
index 0000000..5dfbc3b
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+
+import org.apache.flink.api.common.functions.JoinFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.scala.ClosureCleaner
+import org.apache.flink.api.scala.typeutils.CaseClassSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.TemporalOperator
+import org.apache.flink.streaming.api.function.co.JoinWindowFunction
+import org.apache.flink.streaming.util.keys.PojoKeySelector
+import scala.reflect.ClassTag
+import org.apache.commons.lang.Validate
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
+import org.apache.flink.streaming.api.function.co.CrossWindowFunction
+import org.apache.flink.api.common.functions.CrossFunction
+
+class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
+
+ override def createNextWindowOperator(): StreamCrossOperator.CrossWindow[I1, I2] = {
+
+ val crossWindowFunction = StreamCrossOperator.getCrossWindowFunction(this, (l: I1, r: I2) => (l, r))
+
+ val returnType = new CaseClassTypeInfo[(I1, I2)](
+
+ classOf[(I1, I2)], Seq(input1.getType, input2.getType), Array("_1", "_2")) {
+
+ override def createSerializer: TypeSerializer[(I1, I2)] = {
+ val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
+ for (i <- 0 until getArity) {
+ fieldSerializers(i) = types(i).createSerializer
+ }
+
+ new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
+ override def createInstance(fields: Array[AnyRef]) = {
+ (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
+ }
+ }
+ }
+ }
+
+ val javaStream = input1.connect(input2).addGeneralWindowCombine(
+ crossWindowFunction,
+ returnType, windowSize,
+ slideInterval, timeStamp1, timeStamp2);
+
+ new StreamCrossOperator.CrossWindow[I1, I2](this, javaStream)
+ }
+}
+object StreamCrossOperator {
+
+ private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2], javaStream: JavaStream[(I1, I2)]) extends DataStream[(I1, I2)](javaStream) {
+
+ /**
+ * Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf call will be emitted.
+ *
+ */
+ def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
+
+ val invokable = new CoWindowInvokable[I1, I2, R](
+ clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+
+ javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(), invokable)
+
+ new DataStream[R](javaStream.setType(implicitly[TypeInformation[R]]))
+ }
+ }
+
+ private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2], crossFunction: (I1, I2) => R): CrossWindowFunction[I1, I2, R] = {
+ Validate.notNull(crossFunction, "Join function must not be null.")
+
+ val crossFun = new CrossFunction[I1, I2, R] {
+ val cleanFun = op.input1.clean(crossFunction)
+
+ override def cross(first: I1, second: I2): R = {
+ cleanFun(first, second)
+ }
+ }
+
+ new CrossWindowFunction[I1, I2, R](crossFun)
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/40a3b6bc/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
index 93950a2..fff5e86 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
@@ -42,11 +42,6 @@ class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
object StreamJoinOperator {
- private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
- ClosureCleaner.clean(f, checkSerializable)
- f
- }
-
class JoinWindow[I1, I2](op: StreamJoinOperator[I1, I2]) {
/**
@@ -94,7 +89,7 @@ object StreamJoinOperator {
* Creates a temporal join transformation by defining the second join key.
* The returned transformation wrapes each joined element pair in a tuple2:
* (first, second)
- * To define a custom wrapping, use JoinedStream.with(...)
+ * To define a custom wrapping, use JoinedStream.apply(...)
*/
def equalTo(fields: Int*): JoinedStream[I1, I2] = {
finish(new FieldsKeySelector[I2](fields: _*))
@@ -104,7 +99,7 @@ object StreamJoinOperator {
* Creates a temporal join transformation by defining the second join key.
* The returned transformation wrapes each joined element pair in a tuple2:
* (first, second)
- * To define a custom wrapping, use JoinedStream.with(...)
+ * To define a custom wrapping, use JoinedStream.apply(...)
*/
def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] = {
finish(new PojoKeySelector[I2](op.input2.getType(), (firstField +: otherFields): _*))
@@ -114,7 +109,7 @@ object StreamJoinOperator {
* Creates a temporal join transformation by defining the second join key.
* The returned transformation wrapes each joined element pair in a tuple2:
* (first, second)
- * To define a custom wrapping, use JoinedStream.with(...)
+ * To define a custom wrapping, use JoinedStream.apply(...)
*/
def equalTo[K: TypeInformation](fun: (I2) => K): JoinedStream[I1, I2] = {
val keyType = implicitly[TypeInformation[K]]
@@ -159,6 +154,9 @@ object StreamJoinOperator {
private val op = jp.op
+ /**
+ * Sets a wrapper for the joined elements. For each joined pair, the result of the udf call will be emitted.
+ */
def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
val invokable = new CoWindowInvokable[I1, I2, R](
@@ -175,7 +173,7 @@ object StreamJoinOperator {
val joinFun = new JoinFunction[I1, I2, R] {
- val cleanFun = clean(joinFunction)
+ val cleanFun = jp.op.input1.clean(joinFunction)
override def join(first: I1, second: I2): R = {
cleanFun(first, second)
[36/36] flink git commit: [streaming] Replaced partitionBy with
groupBy + re-added global partitioning
Posted by mb...@apache.org.
[streaming] Replaced partitionBy with groupBy + re-added global partitioning
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/242ff64e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/242ff64e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/242ff64e
Branch: refs/heads/release-0.8
Commit: 242ff64e0513061c3248fe44d0142dee3e84a6b2
Parents: 80ff884
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Jan 7 10:24:00 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 7 13:57:48 2015 +0100
----------------------------------------------------------------------
docs/streaming_guide.md | 6 +-
.../connectors/twitter/TwitterLocal.java | 98 --------------------
.../connectors/twitter/TwitterTopology.java | 92 ++++++++++++++++++
.../streaming/api/datastream/DataStream.java | 67 ++++++-------
.../datastream/SingleOutputStreamOperator.java | 16 ----
.../examples/twitter/TwitterStream.java | 2 +-
.../windowing/MultiplePoliciesExample.java | 9 +-
.../flink/streaming/api/scala/DataStream.scala | 40 ++------
8 files changed, 137 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index c7e7060..7808b9e 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -162,11 +162,11 @@ Usage: `dataStream.shuffle()`
* *Distribute*: Distribute partitioning directs the output data stream to the next operator in a round-robin fashion, achieving a balanced distribution.
Usage: `dataStream.distribute()`
* *Field/Key*: Field/Key partitioning partitions the output data stream based on the hash code of a selected key of the tuples. Data points with the same key are directed to the same operator instance. The user can define keys by field positions (for tuple and array types), field expressions (for Pojo types) and custom keys using the `KeySelector` interface.
-Usage: `dataStream.partitionBy(keys)`
+Usage: `dataStream.groupBy(keys)`
* *Broadcast*: Broadcast partitioning sends the output data stream to all parallel instances of the next operator.
Usage: `dataStream.broadcast()`
- * *Global*: All data points end up at the same operator instance. To achieve this use the parallelism setting of the corresponding operator.
-Usage: `operator.setParallelism(1)`
+ * *Global*: All data are sent to the first instance of the next processing operator. Use this option with care to avoid serious performance bottlenecks.
+Usage: `dataStream.global()`
### Sources
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterLocal.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterLocal.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterLocal.java
deleted file mode 100644
index 3058047..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterLocal.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.twitter;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
-import org.apache.flink.util.Collector;
-import org.apache.sling.commons.json.JSONException;
-
-/**
- * This program demonstrate the use of TwitterSource.
- * Its aim is to count the frequency of the languages of tweets
- */
-public class TwitterLocal {
-
- private static final int PARALLELISM = 1;
- private static final int SOURCE_PARALLELISM = 1;
- private static final int NUMBEROFTWEETS = 100;
-
- /**
- * FlatMapFunction to determine the language of tweets if possible
- */
- public static class SelectLanguageFlatMap extends
- JSONParseFlatMap<String, String> {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Select the language from the incoming JSON text
- */
- @Override
- public void flatMap(String value, Collector<String> out) throws Exception {
- try{
- out.collect(getString(value, "lang"));
- }
- catch (JSONException e){
- out.collect("");
- }
- }
-
- }
-
- public static void main(String[] args) throws Exception {
-
- String path = new String();
-
- if (args != null && args.length == 1) {
- path = args[0];
- } else {
- System.err.println("USAGE:\nTwitterLocal <pathToPropertiesFile>");
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment
- .createLocalEnvironment(PARALLELISM);
-
- DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS))
- .setParallelism(SOURCE_PARALLELISM);
-
-
- DataStream<Tuple2<String, Integer>> dataStream = streamSource
- .flatMap(new SelectLanguageFlatMap())
- .partitionBy(0)
- .map(new MapFunction<String, Tuple2<String, Integer>>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, Integer> map(String value) throws Exception {
- return new Tuple2<String, Integer>(value, 1);
- }
- })
- .groupBy(0)
- .sum(1);
-
- dataStream.print();
-
- env.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
new file mode 100644
index 0000000..4bc6df0
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterTopology.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.twitter;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.json.JSONParseFlatMap;
+import org.apache.flink.util.Collector;
+import org.apache.sling.commons.json.JSONException;
+
+/**
+ * This program demonstrate the use of TwitterSource.
+ * Its aim is to count the frequency of the languages of tweets
+ */
+public class TwitterTopology {
+
+ private static final int NUMBEROFTWEETS = 100;
+
+ /**
+ * FlatMapFunction to determine the language of tweets if possible
+ */
+ public static class SelectLanguageFlatMap extends
+ JSONParseFlatMap<String, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Select the language from the incoming JSON text
+ */
+ @Override
+ public void flatMap(String value, Collector<String> out) throws Exception {
+ try{
+ out.collect(getString(value, "lang"));
+ }
+ catch (JSONException e){
+ out.collect("");
+ }
+ }
+
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String path = new String();
+
+ if (args != null && args.length == 1) {
+ path = args[0];
+ } else {
+ System.err.println("USAGE:\nTwitterLocal <pathToPropertiesFile>");
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ DataStream<String> streamSource = env.addSource(new TwitterSource(path, NUMBEROFTWEETS));
+
+
+ DataStream<Tuple2<String, Integer>> dataStream = streamSource
+ .flatMap(new SelectLanguageFlatMap())
+ .map(new MapFunction<String, Tuple2<String, Integer>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<String, Integer> map(String value) throws Exception {
+ return new Tuple2<String, Integer>(value, 1);
+ }
+ })
+ .groupBy(0)
+ .sum(1);
+
+ dataStream.print();
+
+ env.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index e969647..8e21218 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -67,6 +67,7 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.partitioner.BroadcastPartitioner;
import org.apache.flink.streaming.partitioner.DistributePartitioner;
import org.apache.flink.streaming.partitioner.FieldsPartitioner;
+import org.apache.flink.streaming.partitioner.GlobalPartitioner;
import org.apache.flink.streaming.partitioner.ShufflePartitioner;
import org.apache.flink.streaming.partitioner.StreamPartitioner;
import org.apache.flink.streaming.util.keys.KeySelectorUtil;
@@ -239,7 +240,9 @@ public class DataStream<OUT> {
/**
* Groups the elements of a {@link DataStream} by the given key positions to
* be used with grouped operators like
- * {@link GroupedDataStream#reduce(ReduceFunction)}
+ * {@link GroupedDataStream#reduce(ReduceFunction)}</p> This operator also
+ * affects the partitioning of the stream, by forcing values with the same
+ * key to go to the same processing instance.
*
* @param fields
* The position of the fields on which the {@link DataStream}
@@ -259,7 +262,9 @@ public class DataStream<OUT> {
* is either the name of a public field or a getter method with parentheses
* of the {@link DataStream}S underlying type. A dot can be used to drill
* down into objects, as in {@code "field1.getInnerField2()" }. This method
- * returns an {@link GroupedDataStream}.
+ * returns an {@link GroupedDataStream}.</p> This operator also affects the
+ * partitioning of the stream, by forcing values with the same key to go to
+ * the same processing instance.
*
* @param fields
* One or more field expressions on which the DataStream will be
@@ -275,7 +280,10 @@ public class DataStream<OUT> {
/**
* Groups the elements of a {@link DataStream} by the key extracted by the
* {@link KeySelector} to be used with grouped operators like
- * {@link GroupedDataStream#reduce(ReduceFunction)}
+ * {@link GroupedDataStream#reduce(ReduceFunction)}.
+ * <p/>
+ * This operator also affects the partitioning of the stream, by forcing
+ * values with the same key to go to the same processing instance.
*
* @param keySelector
* The {@link KeySelector} that will be used to extract keys for
@@ -293,42 +301,6 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output is
- * partitioned by the selected fields. This setting only effects the how the
- * outputs will be distributed between the parallel instances of the next
- * processing operator.
- *
- * @param fields
- * The fields to partition by.
- * @return The DataStream with fields partitioning set.
- */
- public DataStream<OUT> partitionBy(int... fields) {
- if (getType() instanceof BasicArrayTypeInfo || getType() instanceof PrimitiveArrayTypeInfo) {
- return partitionBy(new KeySelectorUtil.ArrayKeySelector<OUT>(fields));
- } else {
- return partitionBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
- }
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output is
- * partitioned by the given field expressions. This setting only effects the
- * how the outputs will be distributed between the parallel instances of the
- * next processing operator.
- *
- * @param fields
- * The fields expressions to partition by.
- * @return The DataStream with fields partitioning set.
- */
- public DataStream<OUT> partitionBy(String... fields) {
- return partitionBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
- }
-
- private DataStream<OUT> partitionBy(Keys<OUT> keys) {
- return partitionBy(KeySelectorUtil.getSelectorForKeys(keys, getType()));
- }
-
- /**
- * Sets the partitioning of the {@link DataStream} so that the output is
* partitioned using the given {@link KeySelector}. This setting only
* effects the how the outputs will be distributed between the parallel
* instances of the next processing operator.
@@ -336,7 +308,7 @@ public class DataStream<OUT> {
* @param keySelector
* @return
*/
- public DataStream<OUT> partitionBy(KeySelector<OUT, ?> keySelector) {
+ protected DataStream<OUT> partitionBy(KeySelector<OUT, ?> keySelector) {
return setConnectionType(new FieldsPartitioner<OUT>(clean(keySelector)));
}
@@ -390,6 +362,18 @@ public class DataStream<OUT> {
}
/**
+ * Sets the partitioning of the {@link DataStream} so that the output values
+ * all go to the first instance of the next processing operator. Use this
+ * setting with care since it might cause a serious performance bottleneck
+ * in the application.
+ *
+ * @return The DataStream with shuffle partitioning set.
+ */
+ public DataStream<OUT> global() {
+ return setConnectionType(new GlobalPartitioner<OUT>());
+ }
+
+ /**
* Initiates an iterative part of the program that feeds back data streams.
* The iterative part needs to be closed by calling
* {@link IterativeDataStream#closeWith(DataStream)}. The transformation of
@@ -1007,7 +991,8 @@ public class DataStream<OUT> {
protected <R> DataStream<OUT> addIterationSource(Integer iterationID, long waitTime) {
- DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null, true);
+ DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null,
+ true);
jobGraphBuilder.addIterationHead(returnStream.getId(), this.getId(), iterationID,
degreeOfParallelism, waitTime);
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 4b6edc0..aa85579 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -24,7 +24,6 @@ import org.apache.commons.lang3.SerializationException;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.collector.OutputSelector;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
@@ -151,21 +150,6 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
}
@SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> partitionBy(int... keypositions) {
- return (SingleOutputStreamOperator<OUT, O>) super.partitionBy(keypositions);
- }
-
- @SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> partitionBy(String... fields) {
- return (SingleOutputStreamOperator<OUT, O>) super.partitionBy(fields);
- }
-
- @SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> partitionBy(KeySelector<OUT, ?> keySelector) {
- return (SingleOutputStreamOperator<OUT, O>) super.partitionBy(keySelector);
- }
-
- @SuppressWarnings("unchecked")
public SingleOutputStreamOperator<OUT, O> broadcast() {
return (SingleOutputStreamOperator<OUT, O>) super.broadcast();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
index 08aa5cb..1901475 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/twitter/TwitterStream.java
@@ -71,7 +71,7 @@ public class TwitterStream {
DataStream<Tuple2<String, Integer>> tweets = streamSource
// selecting English tweets and splitting to words
- .flatMap(new SelectEnglishAndTokenizeFlatMap()).partitionBy(0)
+ .flatMap(new SelectEnglishAndTokenizeFlatMap())
// returning (word, 1)
.map(new MapFunction<String, Tuple2<String, Integer>>() {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
index 6f031c3..48783f2 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.examples.windowing;
import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.source.SourceFunction;
@@ -43,7 +44,13 @@ public class MultiplePoliciesExample {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<String> stream = env.addSource(new BasicSource())
- .groupBy(0)
+ .groupBy(new KeySelector<String, String>(){
+ private static final long serialVersionUID = 1L;
+ @Override
+ public String getKey(String value) throws Exception {
+ return value;
+ }
+ })
.window(Count.of(2))
.every(Count.of(3), Count.of(5))
.reduceGroup(new Concat());
http://git-wip-us.apache.org/repos/asf/flink/blob/242ff64e/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index 9b82009..520ce68 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -127,39 +127,6 @@ class DataStream[T](javaStream: JavaStream[T]) {
}
/**
- * Sets the partitioning of the DataStream so that the output is
- * partitioned by the selected fields. This setting only effects the how the outputs will be
- * distributed between the parallel instances of the next processing operator.
- *
- */
- def partitionBy(fields: Int*): DataStream[T] =
- javaStream.partitionBy(fields: _*)
-
- /**
- * Sets the partitioning of the DataStream so that the output is
- * partitioned by the selected fields. This setting only effects the how the outputs will be
- * distributed between the parallel instances of the next processing operator.
- *
- */
- def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
- javaStream.partitionBy(firstField +: otherFields.toArray: _*)
-
- /**
- * Sets the partitioning of the DataStream so that the output is
- * partitioned by the given Key. This setting only effects the how the outputs will be
- * distributed between the parallel instances of the next processing operator.
- *
- */
- def partitionBy[K: TypeInformation](fun: T => K): DataStream[T] = {
-
- val keyExtractor = new KeySelector[T, K] {
- val cleanFun = clean(fun)
- def getKey(in: T) = cleanFun(in)
- }
- javaStream.partitionBy(keyExtractor)
- }
-
- /**
* Sets the partitioning of the DataStream so that the output tuples
* are broadcasted to every parallel instance of the next component. This
* setting only effects the how the outputs will be distributed between the
@@ -167,6 +134,13 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def broadcast: DataStream[T] = javaStream.broadcast()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output values all go to
+ * the first instance of the next processing operator. Use this setting with care
+ * since it might cause a serious performance bottlenect in the application.
+ */
+ def global: DataStream[T] = javaStream.global()
/**
* Sets the partitioning of the DataStream so that the output tuples
[28/36] flink git commit: [docs] Remove files from obolete and unused
figures
Posted by mb...@apache.org.
[docs] Remove files from obolete and unused figures
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e3700c37
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e3700c37
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e3700c37
Branch: refs/heads/release-0.8
Commit: e3700c3707559c19b0f05cb48589ef7311b63e08
Parents: e78c453
Author: Stephan Ewen <se...@apache.org>
Authored: Sun Jan 4 23:09:26 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 00:22:57 2015 +0100
----------------------------------------------------------------------
docs/slot_based_scheduling.jpg | Bin 66562 -> 0 bytes
docs/state_machine.jpg | Bin 85236 -> 0 bytes
2 files changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e3700c37/docs/slot_based_scheduling.jpg
----------------------------------------------------------------------
diff --git a/docs/slot_based_scheduling.jpg b/docs/slot_based_scheduling.jpg
deleted file mode 100644
index 55d2e68..0000000
Binary files a/docs/slot_based_scheduling.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/flink/blob/e3700c37/docs/state_machine.jpg
----------------------------------------------------------------------
diff --git a/docs/state_machine.jpg b/docs/state_machine.jpg
deleted file mode 100644
index a33d4ee..0000000
Binary files a/docs/state_machine.jpg and /dev/null differ
[35/36] flink git commit: [streaming] [scala] scala SocketTextStream
added and minor fixes
Posted by mb...@apache.org.
[streaming] [scala] scala SocketTextStream added and minor fixes
Organized imports for streaming scala examples
Added template parameter for scala streaming iterate
Minor fixes in streaming examples
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/80ff8842
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/80ff8842
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/80ff8842
Branch: refs/heads/release-0.8
Commit: 80ff8842f2743eba34853406c8de7b91230330a0
Parents: b2f6df7
Author: mbalassi <mb...@apache.org>
Authored: Tue Jan 6 22:50:18 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Wed Jan 7 11:51:00 2015 +0100
----------------------------------------------------------------------
.../socket/SocketTextStreamWordCount.java | 59 +++++++------
.../socket/SocketTextStreamWordCount.scala | 91 ++++++++++++++++++++
.../examples/windowing/TopSpeedWindowing.scala | 5 +-
.../scala/examples/windowing/WindowJoin.scala | 11 +--
.../flink/streaming/api/scala/DataStream.scala | 4 +-
5 files changed, 134 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/80ff8842/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
index ec32e9f..e9b60f4 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
@@ -23,11 +23,21 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
/**
- * This example shows an implementation of WordCount with data from socket.
- *
+ * This example shows an implementation of WordCount with data from a text
+ * socket. To run the example make sure that the service providing the text data
+ * is already up and running.
+ *
* <p>
- * Usage: <code>SocketTextStreamWordCount <hostname> <port > <result path></code><br>
- *
+ * To start an example socket text stream on your local machine run netcat from
+ * a command line: <code>nc -lk 9999</code>, where the parameter specifies the
+ * port number.
+ *
+ *
+ * <p>
+ * Usage:
+ * <code>SocketTextStreamWordCount <hostname> <port> <result path></code>
+ * <br>
+ *
* <p>
* This example shows how to:
* <ul>
@@ -35,6 +45,8 @@ import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
* <li>write a simple Flink program,
* <li>write and use user-defined functions.
* </ul>
+ *
+ * @see <a href="www.openbsd.org/cgi-bin/man.cgi?query=nc">netcat</a>
*/
public class SocketTextStreamWordCount {
public static void main(String[] args) throws Exception {
@@ -44,16 +56,18 @@ public class SocketTextStreamWordCount {
}
// set up the execution environment
- final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .getExecutionEnvironment();
// get input data
- DataStream<String> text = env.socketTextStream(hostname, port);
+ DataStream<String> text = env.socketTextStream(hostName, port);
DataStream<Tuple2<String, Integer>> counts =
// split up the lines in pairs (2-tuples) containing: (word,1)
text.flatMap(new Tokenizer())
// group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0).sum(1);
+ .groupBy(0)
+ .sum(1);
if (fileOutput) {
counts.writeAsText(outputPath, 1);
@@ -62,7 +76,7 @@ public class SocketTextStreamWordCount {
}
// execute program
- env.execute("WordCount with SocketTextStream Example");
+ env.execute("WordCount from SocketTextStream Example");
}
// *************************************************************************
@@ -70,30 +84,23 @@ public class SocketTextStreamWordCount {
// *************************************************************************
private static boolean fileOutput = false;
- private static String hostname;
+ private static String hostName;
private static int port;
private static String outputPath;
private static boolean parseParameters(String[] args) {
- if (args.length > 0) {
- // parse input arguments
- if (args.length == 3) {
- fileOutput = true;
- hostname = args[0];
- port = Integer.valueOf(args[1]);
- outputPath = args[2];
- } else if (args.length == 2) {
- hostname = args[0];
- port = Integer.valueOf(args[1]);
- } else {
- System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> <output path>");
- return false;
- }
+ // parse input arguments
+ if (args.length == 3) {
+ fileOutput = true;
+ hostName = args[0];
+ port = Integer.valueOf(args[1]);
+ outputPath = args[2];
+ } else if (args.length == 2) {
+ hostName = args[0];
+ port = Integer.valueOf(args[1]);
} else {
- System.out.println("Executing WordCount example with data from socket.");
- System.out.println(" Provide parameters to connect data source.");
- System.out.println(" Usage: SocketTextStreamWordCount <hostname> <port> <output path>");
+ System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]");
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/flink/blob/80ff8842/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
new file mode 100644
index 0000000..ef5ac9e
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/socket/SocketTextStreamWordCount.scala
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.socket
+
+import org.apache.flink.streaming.api.scala._
+
+/**
+ * This example shows an implementation of WordCount with data from a text socket.
+ * To run the example make sure that the service providing the text data is already up and running.
+ *
+ * To start an example socket text stream on your local machine run netcat from a command line,
+ * where the parameter specifies the port number:
+ *
+ * {{{
+ * nc -lk 9999
+ * }}}
+ *
+ * Usage:
+ * {{{
+ * SocketTextStreamWordCount <hostname> <port> <output path>
+ * }}}
+ *
+ * This example shows how to:
+ *
+ * - use StreamExecutionEnvironment.socketTextStream
+ * - write a simple Flink Streaming program in scala.
+ * - write and use user-defined functions.
+ */
+object SocketTextStreamWordCount {
+
+ def main(args: Array[String]) {
+ if (!parseParameters(args)) {
+ return
+ }
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+
+ //Create streams for names and ages by mapping the inputs to the corresponding objects
+ val text = env.socketTextStream(hostName, port)
+ val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } }
+ .map { (_, 1) }
+ .groupBy(0)
+ .sum(1)
+
+ if (fileOutput) {
+ counts.writeAsCsv(outputPath, 1)
+ } else {
+ counts print
+ }
+
+ env.execute("Scala SocketTextStreamWordCount Example")
+ }
+
+ private def parseParameters(args: Array[String]): Boolean = {
+ if (args.length == 3) {
+ fileOutput = true
+ hostName = args(0)
+ port = args(1).toInt
+ outputPath = args(2)
+ } else if (args.length == 2) {
+ hostName = args(0)
+ port = args(1).toInt
+ } else {
+ System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]")
+ return false
+ }
+ true
+ }
+
+ private var fileOutput: Boolean = false
+ private var hostName: String = null
+ private var port: Int = 0
+ private var outputPath: String = null
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/80ff8842/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
index 3cf5e50..7e9a8b2 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -18,12 +18,11 @@
package org.apache.flink.streaming.scala.examples.windowing
-
import java.util.concurrent.TimeUnit._
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.scala.windowing.{Delta, Time}
-import org.apache.flink.api.scala._
+
import scala.Stream._
import scala.math._
import scala.util.Random
http://git-wip-us.apache.org/repos/asf/flink/blob/80ff8842/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
index e5dc708..3abe199 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
@@ -18,8 +18,7 @@
package org.apache.flink.streaming.scala.examples.windowing
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.scala._
import scala.Stream._
import scala.util.Random
@@ -39,11 +38,13 @@ object WindowJoin {
val names = env.fromCollection(nameStream).map(x => Name(x._1, x._2))
val ages = env.fromCollection(ageStream).map(x => Age(x._1, x._2))
- //Join the two input streams by id on the last second every 2 seconds and create new
+ //Join the two input streams by id on the last 2 seconds every second and create new
//Person objects containing both name and age
val joined =
- names.join(ages).onWindow(1, TimeUnit.SECONDS).every(2, TimeUnit.SECONDS)
- .where("id").equalTo("id") { (n, a) => Person(n.name, a.age) }
+ names.join(ages).onWindow(2, TimeUnit.SECONDS)
+ .every(1, TimeUnit.SECONDS)
+ .where("id")
+ .equalTo("id") { (n, a) => Person(n.name, a.age) }
joined print
http://git-wip-us.apache.org/repos/asf/flink/blob/80ff8842/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index 1eedfc9..9b82009 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -214,8 +214,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*
*/
- def iterate(stepFunction: DataStream[T] => (DataStream[T], DataStream[T]), maxWaitTimeMillis:
- Long = 0): DataStream[T] = {
+ def iterate[R](stepFunction: DataStream[T] => (DataStream[T], DataStream[R]),
+ maxWaitTimeMillis:Long = 0): DataStream[R] = {
val iterativeStream = javaStream.iterate(maxWaitTimeMillis)
val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
[18/36] flink git commit: [scala] [streaming] Added implicit
conversions from java to scala streams
Posted by mb...@apache.org.
[scala] [streaming] Added implicit conversions from java to scala streams
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c34f5a77
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c34f5a77
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c34f5a77
Branch: refs/heads/release-0.8
Commit: c34f5a77026ff491cabc2b5087b1c16da9e276a2
Parents: 1492e96
Author: Gyula Fora <gy...@apache.org>
Authored: Sat Jan 3 19:41:37 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:06:19 2015 +0100
----------------------------------------------------------------------
.../flink/api/scala/streaming/DataStream.scala | 63 ++++++++++----------
.../api/scala/streaming/SplitDataStream.scala | 6 +-
.../scala/streaming/StreamCrossOperator.scala | 3 +-
.../streaming/StreamExecutionEnvironment.scala | 15 ++---
.../scala/streaming/StreamJoinOperator.scala | 3 +-
.../scala/streaming/StreamingConversions.scala | 36 +++++++++++
.../scala/streaming/WindowedDataStream.scala | 15 +++--
7 files changed, 88 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c34f5a77/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index dc1e5b3..546d8a9 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -49,8 +49,8 @@ import org.apache.flink.streaming.api.function.aggregation.SumFunction
import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
-import com.amazonaws.services.cloudfront_2012_03_15.model.InvalidArgumentException
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.api.scala.streaming.StreamingConversions._
class DataStream[T](javaStream: JavaStream[T]) {
@@ -62,7 +62,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
/**
* Sets the degree of parallelism of this operation. This must be greater than 1.
*/
- def setParallelism(dop: Int) = {
+ def setParallelism(dop: Int): DataStream[T] = {
javaStream match {
case ds: SingleOutputStreamOperator[_, _] => ds.setParallelism(dop)
case _ =>
@@ -91,15 +91,14 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def merge(dataStreams: DataStream[T]*): DataStream[T] =
- new DataStream[T](javaStream.merge(dataStreams.map(_.getJavaStream): _*))
+ javaStream.merge(dataStreams.map(_.getJavaStream): _*)
/**
* Groups the elements of a DataStream by the given key positions (for tuple/array types) to
* be used with grouped operators like grouped reduce or grouped aggregations
*
*/
- def groupBy(fields: Int*): DataStream[T] =
- new DataStream[T](javaStream.groupBy(fields: _*))
+ def groupBy(fields: Int*): DataStream[T] = javaStream.groupBy(fields: _*)
/**
* Groups the elements of a DataStream by the given field expressions to
@@ -107,7 +106,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def groupBy(firstField: String, otherFields: String*): DataStream[T] =
- new DataStream[T](javaStream.groupBy(firstField +: otherFields.toArray: _*))
+ javaStream.groupBy(firstField +: otherFields.toArray: _*)
/**
* Groups the elements of a DataStream by the given K key to
@@ -120,7 +119,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
val cleanFun = clean(fun)
def getKey(in: T) = cleanFun(in)
}
- new DataStream[T](javaStream.groupBy(keyExtractor))
+ javaStream.groupBy(keyExtractor)
}
/**
@@ -130,7 +129,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def partitionBy(fields: Int*): DataStream[T] =
- new DataStream[T](javaStream.partitionBy(fields: _*));
+ javaStream.partitionBy(fields: _*)
/**
* Sets the partitioning of the DataStream so that the output is
@@ -139,7 +138,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
- new DataStream[T](javaStream.partitionBy(firstField +: otherFields.toArray: _*))
+ javaStream.partitionBy(firstField +: otherFields.toArray: _*)
/**
* Sets the partitioning of the DataStream so that the output is
@@ -153,7 +152,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
val cleanFun = clean(fun)
def getKey(in: T) = cleanFun(in)
}
- new DataStream[T](javaStream.partitionBy(keyExtractor))
+ javaStream.partitionBy(keyExtractor)
}
/**
@@ -163,7 +162,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* parallel instances of the next processing operator.
*
*/
- def broadcast: DataStream[T] = new DataStream[T](javaStream.broadcast())
+ def broadcast: DataStream[T] = javaStream.broadcast()
/**
* Sets the partitioning of the DataStream so that the output tuples
@@ -172,7 +171,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* processing operator.
*
*/
- def shuffle: DataStream[T] = new DataStream[T](javaStream.shuffle())
+ def shuffle: DataStream[T] = javaStream.shuffle()
/**
* Sets the partitioning of the DataStream so that the output tuples
@@ -182,7 +181,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* instances of the next processing operator.
*
*/
- def forward: DataStream[T] = new DataStream[T](javaStream.forward())
+ def forward: DataStream[T] = javaStream.forward()
/**
* Sets the partitioning of the DataStream so that the output tuples
@@ -191,7 +190,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* the next processing operator.
*
*/
- def distribute: DataStream[T] = new DataStream[T](javaStream.distribute())
+ def distribute: DataStream[T] = javaStream.distribute()
/**
* Initiates an iterative part of the program that creates a loop by feeding
@@ -217,7 +216,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
iterativeStream.closeWith(feedback.getJavaStream)
- new DataStream[T](output.getJavaStream)
+ output
}
/**
@@ -301,8 +300,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
def map(in: T): R = cleanFun(in)
}
- new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]],
- new MapInvokable[T, R](mapper)))
+ javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
}
/**
@@ -313,8 +311,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
throw new NullPointerException("Map function must not be null.")
}
- new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]],
- new MapInvokable[T, R](mapper)))
+ javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
}
/**
@@ -325,8 +322,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
if (flatMapper == null) {
throw new NullPointerException("FlatMap function must not be null.")
}
- new DataStream[R](javaStream.transform("flatMap", implicitly[TypeInformation[R]],
- new FlatMapInvokable[T, R](flatMapper)))
+ javaStream.transform("flatMap", implicitly[TypeInformation[R]],
+ new FlatMapInvokable[T, R](flatMapper))
}
/**
@@ -368,10 +365,10 @@ class DataStream[T](javaStream: JavaStream[T]) {
throw new NullPointerException("Reduce function must not be null.")
}
javaStream match {
- case ds: GroupedDataStream[_] => new DataStream[T](javaStream.transform("reduce",
- javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector())))
- case _ => new DataStream[T](javaStream.transform("reduce", javaStream.getType(),
- new StreamReduceInvokable[T](reducer)))
+ case ds: GroupedDataStream[_] => javaStream.transform("reduce",
+ javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector()))
+ case _ => javaStream.transform("reduce", javaStream.getType(),
+ new StreamReduceInvokable[T](reducer))
}
}
@@ -397,7 +394,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
if (filter == null) {
throw new NullPointerException("Filter function must not be null.")
}
- new DataStream[T](javaStream.filter(filter))
+ javaStream.filter(filter)
}
/**
@@ -426,7 +423,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* window(List(triggers), List(evicters))
*/
def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
- new WindowedDataStream[T](javaStream.window(windowingHelper: _*))
+ javaStream.window(windowingHelper: _*)
/**
* Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
@@ -436,7 +433,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]):
- WindowedDataStream[T] = new WindowedDataStream[T](javaStream.window(triggers, evicters))
+ WindowedDataStream[T] = javaStream.window(triggers, evicters)
/**
*
@@ -445,7 +442,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* SplitDataStream.
*/
def split(selector: OutputSelector[T]): SplitDataStream[T] = javaStream match {
- case op: SingleOutputStreamOperator[_, _] => new SplitDataStream[T](op.split(selector))
+ case op: SingleOutputStreamOperator[_, _] => op.split(selector)
case _ =>
throw new UnsupportedOperationException("Operator " + javaStream.toString + " can not be " +
"split.")
@@ -503,7 +500,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* written.
*
*/
- def print(): DataStream[T] = new DataStream[T](javaStream.print())
+ def print(): DataStream[T] = javaStream.print()
/**
* Writes a DataStream to the file specified by path in text format. The
@@ -513,7 +510,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def writeAsText(path: String, millis: Long = 0): DataStream[T] =
- new DataStream[T](javaStream.writeAsText(path, millis))
+ javaStream.writeAsText(path, millis)
/**
* Writes a DataStream to the file specified by path in text format. The
@@ -523,7 +520,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def writeAsCsv(path: String, millis: Long = 0): DataStream[T] =
- new DataStream[T](javaStream.writeAsCsv(path, millis))
+ javaStream.writeAsCsv(path, millis)
/**
* Adds the given sink to this DataStream. Only streams with sinks added
@@ -532,7 +529,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] =
- new DataStream[T](javaStream.addSink(sinkFuntion))
+ javaStream.addSink(sinkFuntion)
/**
* Adds the given sink to this DataStream. Only streams with sinks added
http://git-wip-us.apache.org/repos/asf/flink/blob/c34f5a77/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
index 82a5c70..f61e34b 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
@@ -20,6 +20,7 @@ package org.apache.flink.api.scala.streaming
import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+import org.apache.flink.api.scala.streaming.StreamingConversions._
/**
* The SplitDataStream represents an operator that has been split using an
@@ -39,12 +40,11 @@ class SplitDataStream[T](javaStream: SplitJavaStream[T]) {
/**
* Sets the output names for which the next operator will receive values.
*/
- def select(outputNames: String*): DataStream[T] =
- new DataStream[T](javaStream.select(outputNames: _*))
+ def select(outputNames: String*): DataStream[T] = javaStream.select(outputNames: _*)
/**
* Selects all output names from a split data stream.
*/
- def selectAll(): DataStream[T] = new DataStream[T](javaStream.selectAll())
+ def selectAll(): DataStream[T] = javaStream.selectAll()
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c34f5a77/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
index e9010c8..cac2927 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
@@ -33,6 +33,7 @@ import org.apache.flink.api.common.functions.CrossFunction
import org.apache.flink.api.scala.typeutils.CaseClassSerializer
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
+import org.apache.flink.api.scala.streaming.StreamingConversions._
class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
@@ -88,7 +89,7 @@ object StreamCrossOperator {
javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
invokable)
- new DataStream[R](javaStream.setType(implicitly[TypeInformation[R]]))
+ javaStream.setType(implicitly[TypeInformation[R]])
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c34f5a77/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index a7a471f..9c66b24 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -29,6 +29,7 @@ import org.apache.flink.streaming.api.function.source.FromElementsFunction
import org.apache.flink.streaming.api.function.source.SourceFunction
import scala.collection.JavaConversions._
import org.apache.flink.util.Collector
+import org.apache.flink.api.scala.streaming.StreamingConversions._
class StreamExecutionEnvironment(javaEnv: JavaEnv) {
@@ -82,7 +83,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*
*/
def readTextFile(filePath: String): DataStream[String] =
- new DataStream[String](javaEnv.readTextFile(filePath))
+ javaEnv.readTextFile(filePath)
/**
* Creates a DataStream that represents the Strings produced by reading the
@@ -91,8 +92,8 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
* testing a topology.
*
*/
- def readTextStream(StreamPath: String): DataStream[String] =
- new DataStream[String](javaEnv.readTextStream(StreamPath))
+ def readTextStream(StreamPath: String): DataStream[String] =
+ javaEnv.readTextStream(StreamPath)
/**
* Creates a new DataStream that contains the strings received infinitely
@@ -101,7 +102,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*
*/
def socketTextStream(hostname: String, port: Int, delimiter: Char): DataStream[String] =
- new DataStream[String](javaEnv.socketTextStream(hostname, port, delimiter))
+ javaEnv.socketTextStream(hostname, port, delimiter)
/**
* Creates a new DataStream that contains the strings received infinitely
@@ -110,7 +111,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*
*/
def socketTextStream(hostname: String, port: Int): DataStream[String] =
- new DataStream[String](javaEnv.socketTextStream(hostname, port))
+ javaEnv.socketTextStream(hostname, port)
/**
* Creates a new DataStream that contains a sequence of numbers.
@@ -151,7 +152,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
new SourceInvokable[T](new FromElementsFunction[T](scala.collection.JavaConversions
.asJavaCollection(data))), null, typeInfo,
"source", 1);
- new DataStream(returnStream)
+ returnStream
}
/**
@@ -163,7 +164,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
Validate.notNull(function, "Function must not be null.")
val cleanFun = StreamExecutionEnvironment.clean(function)
val typeInfo = implicitly[TypeInformation[T]]
- new DataStream[T](javaEnv.addSource(cleanFun, typeInfo))
+ javaEnv.addSource(cleanFun, typeInfo)
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/c34f5a77/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
index 4ed5082..8d8a0b0 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
@@ -34,6 +34,7 @@ import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.util.keys.KeySelectorUtil
import org.apache.flink.api.java.operators.Keys
import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
+import org.apache.flink.api.scala.streaming.StreamingConversions._
class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
@@ -178,7 +179,7 @@ object StreamJoinOperator {
javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
invokable)
- new DataStream[R](javaStream.setType(implicitly[TypeInformation[R]]))
+ javaStream.setType(implicitly[TypeInformation[R]])
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c34f5a77/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
new file mode 100644
index 0000000..a34d0dc
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
+import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+
+object StreamingConversions {
+
+ implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
+ new DataStream[R](javaStream)
+
+ implicit def javaToScalaWindowedStream[R](javaWStream: JavaWStream[R]): WindowedDataStream[R] =
+ new WindowedDataStream[R](javaWStream)
+
+ implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
+ new SplitDataStream[R](javaStream)
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/c34f5a77/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
index e33368c..2f9c792 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -41,6 +41,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
import org.apache.flink.streaming.api.function.aggregation.SumFunction
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
+import org.apache.flink.api.scala.streaming.StreamingConversions._
class WindowedDataStream[T](javaStream: JavaWStream[T]) {
@@ -50,7 +51,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
* the window.
*/
def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
- new WindowedDataStream[T](javaStream.every(windowingHelper: _*))
+ javaStream.every(windowingHelper: _*)
/**
* Groups the elements of the WindowedDataStream using the given
@@ -61,8 +62,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
* DataStream.window(...) operator on an already grouped data stream.
*
*/
- def groupBy(fields: Int*): WindowedDataStream[T] =
- new WindowedDataStream[T](javaStream.groupBy(fields: _*))
+ def groupBy(fields: Int*): WindowedDataStream[T] = javaStream.groupBy(fields: _*)
/**
* Groups the elements of the WindowedDataStream using the given
@@ -74,8 +74,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
*
*/
def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
- new WindowedDataStream[T](javaStream.groupBy(
- firstField +: otherFields.toArray: _*))
+ javaStream.groupBy(firstField +: otherFields.toArray: _*)
/**
* Groups the elements of the WindowedDataStream using the given
@@ -92,7 +91,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
val cleanFun = clean(fun)
def getKey(in: T) = cleanFun(in)
}
- new WindowedDataStream[T](javaStream.groupBy(keyExtractor))
+ javaStream.groupBy(keyExtractor)
}
/**
@@ -104,7 +103,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
if (reducer == null) {
throw new NullPointerException("Reduce function must not be null.")
}
- new DataStream[T](javaStream.reduce(reducer))
+ javaStream.reduce(reducer)
}
/**
@@ -136,7 +135,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.")
}
- new DataStream[R](javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]]))
+ javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]])
}
/**
[23/36] flink git commit: [streaming] Updated connector type handling
to suport generic classes by GenericSourceFunction interface
Posted by mb...@apache.org.
[streaming] Updated connector type handling to suport generic classes by GenericSourceFunction interface
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/380934b1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/380934b1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/380934b1
Branch: refs/heads/release-0.8
Commit: 380934b1beb3a27a2ca3b99bc03ca7a10905589d
Parents: e0fd9df
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Dec 25 18:00:26 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:08:54 2015 +0100
----------------------------------------------------------------------
.../streaming/connectors/ConnectorSource.java | 42 ++++++++++++++++++++
.../streaming/connectors/flume/FlumeSink.java | 20 +++++-----
.../streaming/connectors/flume/FlumeSource.java | 15 ++++---
.../connectors/flume/FlumeTopology.java | 8 ++--
.../streaming/connectors/kafka/KafkaSink.java | 8 ++--
.../streaming/connectors/kafka/KafkaSource.java | 15 ++++---
.../connectors/kafka/KafkaTopology.java | 21 ++--------
.../streaming/connectors/rabbitmq/RMQSink.java | 10 ++---
.../connectors/rabbitmq/RMQSource.java | 16 ++++----
.../connectors/rabbitmq/RMQTopology.java | 8 ++--
.../connectors/util/DeserializationSchema.java | 42 ++++++++++++++++++++
.../connectors/util/DeserializationScheme.java | 42 --------------------
.../streaming/connectors/util/RawSchema.java | 39 ++++++++++++++++++
.../streaming/connectors/util/RawScheme.java | 39 ------------------
.../connectors/util/SerializationSchema.java | 33 +++++++++++++++
.../connectors/util/SerializationScheme.java | 33 ---------------
.../connectors/util/SimpleStringSchema.java | 40 +++++++++++++++++++
.../connectors/util/SimpleStringScheme.java | 40 -------------------
.../environment/StreamExecutionEnvironment.java | 10 ++++-
.../function/source/GenericSourceFunction.java | 25 ++++++++++++
20 files changed, 282 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
new file mode 100644
index 0000000..1623943
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.function.source.GenericSourceFunction;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
+
+public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements
+ GenericSourceFunction<OUT> {
+
+ private static final long serialVersionUID = 1L;
+ protected DeserializationSchema<OUT> schema;
+
+ public ConnectorSource(DeserializationSchema<OUT> schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public TypeInformation<OUT> getType() {
+ return TypeExtractor.createTypeInfo(DeserializationSchema.class, schema.getClass(), 0,
+ null, null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
index 3e68c45..8a2f2b8 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
@@ -17,9 +17,10 @@
package org.apache.flink.streaming.connectors.flume;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
-import org.apache.flink.streaming.connectors.util.SerializationScheme;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
@@ -38,12 +39,12 @@ public class FlumeSink<IN> extends RichSinkFunction<IN> {
boolean initDone = false;
String host;
int port;
- SerializationScheme<IN, byte[]> scheme;
+ SerializationSchema<IN, byte[]> scheme;
- public FlumeSink(String host, int port, SerializationScheme<IN, byte[]> scheme) {
+ public FlumeSink(String host, int port, SerializationSchema<IN, byte[]> schema) {
this.host = host;
this.port = port;
- this.scheme = scheme;
+ this.scheme = schema;
}
/**
@@ -56,11 +57,6 @@ public class FlumeSink<IN> extends RichSinkFunction<IN> {
@Override
public void invoke(IN value) {
- if (!initDone) {
- client = new FlinkRpcClientFacade();
- client.init(host, port);
- }
-
byte[] data = scheme.serialize(value);
client.sendDataToFlume(data);
@@ -136,4 +132,10 @@ public class FlumeSink<IN> extends RichSinkFunction<IN> {
public void close() {
client.client.close();
}
+
+ @Override
+ public void open(Configuration config) {
+ client = new FlinkRpcClientFacade();
+ client.init(host, port);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
index ceb2be8..4f6ec2d 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
@@ -20,8 +20,8 @@ package org.apache.flink.streaming.connectors.flume;
import java.util.List;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.source.ParallelSourceFunction;
-import org.apache.flink.streaming.connectors.util.DeserializationScheme;
+import org.apache.flink.streaming.connectors.ConnectorSource;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
import org.apache.flink.util.Collector;
import org.apache.flume.Context;
import org.apache.flume.channel.ChannelProcessor;
@@ -29,18 +29,17 @@ import org.apache.flume.source.AvroSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.Status;
-public class FlumeSource<OUT> implements ParallelSourceFunction<OUT> {
+public class FlumeSource<OUT> extends ConnectorSource<OUT> {
private static final long serialVersionUID = 1L;
String host;
String port;
- DeserializationScheme<OUT> scheme;
volatile boolean finished = false;
- FlumeSource(String host, int port, DeserializationScheme<OUT> scheme) {
+ FlumeSource(String host, int port, DeserializationSchema<OUT> deserializationSchema) {
+ super(deserializationSchema);
this.host = host;
this.port = Integer.toString(port);
- this.scheme = scheme;
}
public class MyAvroSource extends AvroSource {
@@ -87,9 +86,9 @@ public class FlumeSource<OUT> implements ParallelSourceFunction<OUT> {
*/
private void collect(AvroFlumeEvent avroEvent) {
byte[] b = avroEvent.getBody().array();
- OUT out = FlumeSource.this.scheme.deserialize(b);
+ OUT out = FlumeSource.this.schema.deserialize(b);
- if (scheme.isEndOfStream(out)) {
+ if (schema.isEndOfStream(out)) {
FlumeSource.this.finished = true;
this.stop();
FlumeSource.this.notifyAll();
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
index 366b1a5..3cfd7d4 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
@@ -19,8 +19,8 @@ package org.apache.flink.streaming.connectors.flume;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.util.SerializationScheme;
-import org.apache.flink.streaming.connectors.util.SimpleStringScheme;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
+import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
public class FlumeTopology {
@@ -30,13 +30,13 @@ public class FlumeTopology {
@SuppressWarnings("unused")
DataStream<String> dataStream1 = env.addSource(
- new FlumeSource<String>("localhost", 41414, new SimpleStringScheme())).addSink(
+ new FlumeSource<String>("localhost", 41414, new SimpleStringSchema())).addSink(
new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
env.execute();
}
- public static class StringToByteSerializer implements SerializationScheme<String, byte[]> {
+ public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
index 71e25f2..9bb87a0 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
@@ -24,7 +24,7 @@ import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
-import org.apache.flink.streaming.connectors.util.SerializationScheme;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
public class KafkaSink<IN, OUT> extends RichSinkFunction<IN> {
private static final long serialVersionUID = 1L;
@@ -34,13 +34,13 @@ public class KafkaSink<IN, OUT> extends RichSinkFunction<IN> {
private String topicId;
private String brokerAddr;
private boolean initDone = false;
- private SerializationScheme<IN, OUT> scheme;
+ private SerializationSchema<IN, OUT> scheme;
public KafkaSink(String topicId, String brokerAddr,
- SerializationScheme<IN, OUT> serializationScheme) {
+ SerializationSchema<IN, OUT> serializationSchema) {
this.topicId = topicId;
this.brokerAddr = brokerAddr;
- this.scheme = serializationScheme;
+ this.scheme = serializationSchema;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
index 49e9144..7328500 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
@@ -29,27 +29,26 @@ import kafka.javaapi.consumer.ConsumerConnector;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.connectors.util.DeserializationScheme;
+import org.apache.flink.streaming.connectors.ConnectorSource;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
import org.apache.flink.util.Collector;
-public class KafkaSource<OUT> extends RichParallelSourceFunction<OUT> {
+public class KafkaSource<OUT> extends ConnectorSource<OUT> {
private static final long serialVersionUID = 1L;
private final String zkQuorum;
private final String groupId;
private final String topicId;
private ConsumerConnector consumer;
- private DeserializationScheme<OUT> scheme;
OUT outTuple;
public KafkaSource(String zkQuorum, String groupId, String topicId,
- DeserializationScheme<OUT> deserializationScheme) {
+ DeserializationSchema<OUT> deserializationSchema) {
+ super(deserializationSchema);
this.zkQuorum = zkQuorum;
this.groupId = groupId;
this.topicId = topicId;
- this.scheme = deserializationScheme;
}
/**
@@ -81,8 +80,8 @@ public class KafkaSource<OUT> extends RichParallelSourceFunction<OUT> {
ConsumerIterator<byte[], byte[]> it = stream.iterator();
while (it.hasNext()) {
- OUT out = scheme.deserialize(it.next().message());
- if (scheme.isEndOfStream(out)) {
+ OUT out = schema.deserialize(it.next().message());
+ if (schema.isEndOfStream(out)) {
break;
}
collector.collect(out);
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
index 16f123e..7801d56 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
@@ -19,15 +19,11 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.connectors.util.SimpleStringScheme;
+import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class KafkaTopology {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaTopology.class);
public static final class MySource implements SourceFunction<String> {
private static final long serialVersionUID = 1L;
@@ -42,17 +38,6 @@ public class KafkaTopology {
}
}
- public static final class MyKafkaPrintSink implements SinkFunction<String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(String value) {
- if (LOG.isInfoEnabled()) {
- LOG.info("String: <{}> arrived from Kafka", value);
- }
- }
- }
-
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
@@ -61,11 +46,11 @@ public class KafkaTopology {
DataStream<String> stream1 = env
.addSource(
new KafkaSource<String>("localhost:2181", "group", "test",
- new SimpleStringScheme())).addSink(new MyKafkaPrintSink());
+ new SimpleStringSchema())).print();
@SuppressWarnings("unused")
DataStream<String> stream2 = env.addSource(new MySource()).addSink(
- new KafkaSink<String, String>("test", "localhost:9092", new SimpleStringScheme()));
+ new KafkaSink<String, String>("test", "localhost:9092", new SimpleStringSchema()));
env.execute();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
index c4f4615..38c4f5f 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
-import org.apache.flink.streaming.connectors.util.SerializationScheme;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,12 +39,12 @@ public class RMQSink<IN> extends RichSinkFunction<IN> {
private transient ConnectionFactory factory;
private transient Connection connection;
private transient Channel channel;
- private SerializationScheme<IN, byte[]> scheme;
+ private SerializationSchema<IN, byte[]> scheme;
- public RMQSink(String HOST_NAME, String QUEUE_NAME, SerializationScheme<IN, byte[]> scheme) {
+ public RMQSink(String HOST_NAME, String QUEUE_NAME, SerializationSchema<IN, byte[]> schema) {
this.HOST_NAME = HOST_NAME;
this.QUEUE_NAME = QUEUE_NAME;
- this.scheme = scheme;
+ this.scheme = schema;
}
/**
@@ -56,6 +56,7 @@ public class RMQSink<IN> extends RichSinkFunction<IN> {
try {
connection = factory.newConnection();
channel = connection.createChannel();
+ channel.queueDeclare(QUEUE_NAME, false, false, false, null);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -71,7 +72,6 @@ public class RMQSink<IN> extends RichSinkFunction<IN> {
@Override
public void invoke(IN value) {
try {
- channel.queueDeclare(QUEUE_NAME, false, false, false, null);
byte[] msg = scheme.serialize(value);
channel.basicPublish("", QUEUE_NAME, null, msg);
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
index 240deab..7ce864e 100755
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
@@ -21,8 +21,8 @@ import java.io.IOException;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
-import org.apache.flink.streaming.connectors.util.DeserializationScheme;
+import org.apache.flink.streaming.connectors.ConnectorSource;
+import org.apache.flink.streaming.connectors.util.DeserializationSchema;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,7 +32,7 @@ import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.QueueingConsumer;
-public class RMQSource<OUT> extends RichParallelSourceFunction<OUT> {
+public class RMQSource<OUT> extends ConnectorSource<OUT> {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(RMQSource.class);
@@ -46,11 +46,11 @@ public class RMQSource<OUT> extends RichParallelSourceFunction<OUT> {
private transient QueueingConsumer consumer;
private transient QueueingConsumer.Delivery delivery;
- private DeserializationScheme<OUT> scheme;
-
OUT out;
- public RMQSource(String HOST_NAME, String QUEUE_NAME, DeserializationScheme<OUT> scheme) {
+ public RMQSource(String HOST_NAME, String QUEUE_NAME,
+ DeserializationSchema<OUT> deserializationSchema) {
+ super(deserializationSchema);
this.HOST_NAME = HOST_NAME;
this.QUEUE_NAME = QUEUE_NAME;
}
@@ -92,8 +92,8 @@ public class RMQSource<OUT> extends RichParallelSourceFunction<OUT> {
}
}
- out = scheme.deserialize(delivery.getBody());
- if (scheme.isEndOfStream(out)) {
+ out = schema.deserialize(delivery.getBody());
+ if (schema.isEndOfStream(out)) {
break;
} else {
collector.collect(out);
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
index 4f82b24..a6ca9ae 100755
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
@@ -19,8 +19,8 @@ package org.apache.flink.streaming.connectors.rabbitmq;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.util.SerializationScheme;
-import org.apache.flink.streaming.connectors.util.SimpleStringScheme;
+import org.apache.flink.streaming.connectors.util.SerializationSchema;
+import org.apache.flink.streaming.connectors.util.SimpleStringSchema;
public class RMQTopology {
@@ -30,7 +30,7 @@ public class RMQTopology {
@SuppressWarnings("unused")
DataStream<String> dataStream1 = env.addSource(
- new RMQSource<String>("localhost", "hello", new SimpleStringScheme())).print();
+ new RMQSource<String>("localhost", "hello", new SimpleStringSchema())).print();
@SuppressWarnings("unused")
DataStream<String> dataStream2 = env.fromElements("one", "two", "three", "four", "five",
@@ -40,7 +40,7 @@ public class RMQTopology {
env.execute();
}
- public static class StringToByteSerializer implements SerializationScheme<String, byte[]> {
+ public static class StringToByteSerializer implements SerializationSchema<String, byte[]> {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
new file mode 100644
index 0000000..4507a1d
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationSchema.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+import java.io.Serializable;
+
+public interface DeserializationSchema<T> extends Serializable {
+
+ /**
+ * Deserializes the incoming data.
+ *
+ * @param message
+ * The incoming message in a byte array
+ * @return The deserialized message in the required format.
+ */
+ public T deserialize(byte[] message);
+
+ /**
+ * Method to decide whether the element signals the end of the stream. If
+ * true is returned the element won't be emitted
+ *
+ * @param nextElement
+ * The element to test for end signal
+ * @return The end signal, if true the stream shuts down
+ */
+ public boolean isEndOfStream(T nextElement);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java
deleted file mode 100644
index 43420a3..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-import java.io.Serializable;
-
-public interface DeserializationScheme<T> extends Serializable {
-
- /**
- * Deserializes the incoming data.
- *
- * @param message
- * The incoming message in a byte array
- * @return The deserialized message in the required format.
- */
- public T deserialize(byte[] message);
-
- /**
- * Method to decide whether the element signals the end of the stream. If
- * true is returned the element won't be emitted
- *
- * @param nextElement
- * The element to test for end signal
- * @return The end signal, if true the stream shuts down
- */
- public boolean isEndOfStream(T nextElement);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
new file mode 100644
index 0000000..29c749a
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawSchema.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+public class RawSchema implements DeserializationSchema<byte[]>,
+ SerializationSchema<byte[], byte[]> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public byte[] deserialize(byte[] message) {
+ return message;
+ }
+
+ @Override
+ public boolean isEndOfStream(byte[] nextElement) {
+ return false;
+ }
+
+ @Override
+ public byte[] serialize(byte[] element) {
+ return element;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java
deleted file mode 100644
index 34db00e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-public class RawScheme implements DeserializationScheme<byte[]>,
- SerializationScheme<byte[], byte[]> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public byte[] deserialize(byte[] message) {
- return message;
- }
-
- @Override
- public boolean isEndOfStream(byte[] nextElement) {
- return false;
- }
-
- @Override
- public byte[] serialize(byte[] element) {
- return element;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
new file mode 100644
index 0000000..7c32312
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationSchema.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+import java.io.Serializable;
+
+public interface SerializationSchema<T,R> extends Serializable {
+
+ /**
+ * Serializes the incoming element to a specified type.
+ *
+ * @param element
+ * The incoming element to be serialized
+ * @return The serialized element.
+ */
+ public R serialize(T element);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java
deleted file mode 100644
index bb25885..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-import java.io.Serializable;
-
-public interface SerializationScheme<T,R> extends Serializable {
-
- /**
- * Serializes the incoming element to a specified type.
- *
- * @param element
- * The incoming element to be serialized
- * @return The serialized element.
- */
- public R serialize(T element);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
new file mode 100644
index 0000000..4b21580
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringSchema.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+public class SimpleStringSchema implements DeserializationSchema<String>,
+ SerializationSchema<String, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String deserialize(byte[] message) {
+ return new String(message);
+ }
+
+ @Override
+ public boolean isEndOfStream(String nextElement) {
+ return false;
+ }
+
+ @Override
+ public String serialize(String element) {
+ return element;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java
deleted file mode 100644
index 9d1ce6c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.util;
-
-public class SimpleStringScheme implements DeserializationScheme<String>,
- SerializationScheme<String, String> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String deserialize(byte[] message) {
- return new String(message);
- }
-
- @Override
- public boolean isEndOfStream(String nextElement) {
- return false;
- }
-
- @Override
- public String serialize(String element) {
- return element;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index c7792a4..6fa9d74 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -40,6 +40,7 @@ import org.apache.flink.streaming.api.function.source.FileSourceFunction;
import org.apache.flink.streaming.api.function.source.FileStreamFunction;
import org.apache.flink.streaming.api.function.source.FromElementsFunction;
import org.apache.flink.streaming.api.function.source.GenSequenceFunction;
+import org.apache.flink.streaming.api.function.source.GenericSourceFunction;
import org.apache.flink.streaming.api.function.source.ParallelSourceFunction;
import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
@@ -414,12 +415,17 @@ public abstract class StreamExecutionEnvironment {
* type of the returned stream
* @return the data stream constructed
*/
+ @SuppressWarnings("unchecked")
private <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
TypeInformation<OUT> outTypeInfo, String sourceName) {
if (outTypeInfo == null) {
- outTypeInfo = TypeExtractor.createTypeInfo(SourceFunction.class, function.getClass(),
- 0, null, null);
+ if (function instanceof GenericSourceFunction) {
+ outTypeInfo = ((GenericSourceFunction<OUT>) function).getType();
+ } else {
+ outTypeInfo = TypeExtractor.createTypeInfo(SourceFunction.class,
+ function.getClass(), 0, null, null);
+ }
}
boolean isParallel = function instanceof ParallelSourceFunction;
http://git-wip-us.apache.org/repos/asf/flink/blob/380934b1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
new file mode 100644
index 0000000..664d39a
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenericSourceFunction.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+
+public interface GenericSourceFunction<T> {
+
+ public TypeInformation<T> getType();
+}
[33/36] flink git commit: [scala] [streaming] Added package file for
streaming scala api typeinfo implicits and conversions
Posted by mb...@apache.org.
[scala] [streaming] Added package file for streaming scala api typeinfo implicits and conversions
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/544b9248
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/544b9248
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/544b9248
Branch: refs/heads/release-0.8
Commit: 544b9248ae78d31caf56a2a28726bb967bd36087
Parents: acd2d60
Author: Gyula Fora <gy...@apache.org>
Authored: Tue Jan 6 12:05:22 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 15:47:53 2015 +0100
----------------------------------------------------------------------
.../examples/windowing/TopSpeedWindowing.scala | 3 +-
.../scala/examples/windowing/WindowJoin.scala | 3 +-
.../api/scala/ConnectedDataStream.scala | 3 +-
.../flink/streaming/api/scala/DataStream.scala | 3 --
.../streaming/api/scala/SplitDataStream.scala | 2 -
.../api/scala/StreamCrossOperator.scala | 2 -
.../api/scala/StreamExecutionEnvironment.scala | 4 +-
.../api/scala/StreamJoinOperator.scala | 1 -
.../api/scala/StreamingConversions.scala | 40 -----------------
.../api/scala/WindowedDataStream.scala | 2 -
.../flink/streaming/api/scala/package.scala | 46 ++++++++++++++++++++
.../streaming/api/scala/windowing/Time.scala | 1 -
12 files changed, 50 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
index 54324e5..a86b953 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -21,8 +21,7 @@ package org.apache.flink.streaming.scala.examples.windowing
import java.util.concurrent.TimeUnit._
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector
import scala.math.{max, min}
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
index b109330..da69774 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
@@ -18,8 +18,7 @@
package org.apache.flink.streaming.scala.examples.windowing
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector
import scala.util.Random
import java.util.concurrent.TimeUnit
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
index 65b04b2..b9a8794 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
@@ -25,12 +25,11 @@ import scala.reflect.ClassTag
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.datastream.{ConnectedDataStream => JavaCStream}
import org.apache.flink.streaming.api.function.co.{ CoFlatMapFunction, CoMapFunction, CoReduceFunction, CoWindowFunction }
import org.apache.flink.streaming.api.invokable.operator.co.{ CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable }
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.scala.StreamingConversions._
import org.apache.flink.util.Collector
class ConnectedDataStream[IN1, IN2](javaStream: JavaCStream[IN1, IN2]) {
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index 93444a5..1eedfc9 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -18,10 +18,8 @@
package org.apache.flink.streaming.api.scala
-import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream,
SingleOutputStreamOperator, GroupedDataStream}
-import org.apache.flink.api.common.typeinfo.TypeInformation
import scala.reflect.ClassTag
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.functions.MapFunction
@@ -47,7 +45,6 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.streaming.api.scala.StreamingConversions._
import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
class DataStream[T](javaStream: JavaStream[T]) {
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
index a34248b..f8daa2e 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
@@ -18,9 +18,7 @@
package org.apache.flink.streaming.api.scala
-import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
-import org.apache.flink.streaming.api.scala.StreamingConversions._
/**
* The SplitDataStream represents an operator that has been split using an
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
index 67ac8ae..a8d4ac2 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
@@ -23,14 +23,12 @@ import org.apache.commons.lang.Validate
import org.apache.flink.api.common.functions.CrossFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.api.scala._
import org.apache.flink.api.scala.typeutils.CaseClassSerializer
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream}
import org.apache.flink.streaming.api.function.co.CrossWindowFunction
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.scala.StreamingConversions._
import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
import java.util.concurrent.TimeUnit
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
index eb34b80..b9006de 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -19,14 +19,12 @@
package org.apache.flink.streaming.api.scala
import scala.reflect.ClassTag
-
import org.apache.commons.lang.Validate
import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaEnv}
import org.apache.flink.streaming.api.function.source.{ FromElementsFunction, SourceFunction }
-import org.apache.flink.streaming.api.scala.StreamingConversions.javaToScalaStream
import org.apache.flink.util.Collector
+import org.apache.flink.api.scala.ClosureCleaner
class StreamExecutionEnvironment(javaEnv: JavaEnv) {
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
index 0d95c5e..0457c18 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
@@ -32,7 +32,6 @@ import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
import org.apache.flink.streaming.api.function.co.JoinWindowFunction
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.scala.StreamingConversions.javaToScalaStream
import org.apache.flink.streaming.util.keys.KeySelectorUtil
import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
import java.util.concurrent.TimeUnit
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala
deleted file mode 100644
index 08ebe03..0000000
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.scala
-
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
-import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
-import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
-
-object StreamingConversions {
-
- implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
- new DataStream[R](javaStream)
-
- implicit def javaToScalaWindowedStream[R](javaWStream: JavaWStream[R]): WindowedDataStream[R] =
- new WindowedDataStream[R](javaWStream)
-
- implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
- new SplitDataStream[R](javaStream)
-
- implicit def javaToScalaConnectedStream[IN1, IN2](javaStream: JavaConStream[IN1, IN2]):
- ConnectedDataStream[IN1, IN2] = new ConnectedDataStream[IN1, IN2](javaStream)
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
index 9b7edd5..8f6fcbb 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -27,13 +27,11 @@ import org.apache.flink.api.common.functions.ReduceFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.api.scala._
import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
import org.apache.flink.streaming.api.datastream.{WindowedDataStream => JavaWStream}
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
import org.apache.flink.streaming.api.function.aggregation.SumFunction
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.scala.StreamingConversions._
import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
import org.apache.flink.streaming.api.windowing.helper._
import org.apache.flink.util.Collector
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
new file mode 100644
index 0000000..f4c6bcf
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api
+
+import _root_.scala.reflect.ClassTag
+import language.experimental.macros
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.scala.typeutils.{CaseClassTypeInfo, TypeUtils}
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
+import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
+
+package object scala {
+ // We have this here so that we always have generated TypeInformationS when
+ // using the Scala API
+ implicit def createTypeInformation[T]: TypeInformation[T] = macro TypeUtils.createTypeInfo[T]
+
+ implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
+ new DataStream[R](javaStream)
+
+ implicit def javaToScalaWindowedStream[R](javaWStream: JavaWStream[R]): WindowedDataStream[R] =
+ new WindowedDataStream[R](javaWStream)
+
+ implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
+ new SplitDataStream[R](javaStream)
+
+ implicit def javaToScalaConnectedStream[IN1, IN2](javaStream: JavaConStream[IN1, IN2]):
+ ConnectedDataStream[IN1, IN2] = new ConnectedDataStream[IN1, IN2](javaStream)
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/544b9248/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
index 457d10d..16d3d20 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
@@ -21,7 +21,6 @@ package org.apache.flink.streaming.api.scala.windowing
import java.util.concurrent.TimeUnit
import org.apache.flink.streaming.api.windowing.helper.{ Time => JavaTime }
-import org.apache.flink.api.scala.ClosureCleaner
import org.apache.commons.net.ntp.TimeStamp
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean;
import org.apache.flink.streaming.api.windowing.helper.Timestamp
[15/36] flink git commit: [streaming] Streaming API grouping rework
to use batch api Keys
Posted by mb...@apache.org.
[streaming] Streaming API grouping rework to use batch api Keys
Conflicts:
flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/ClusterUtil.java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/59952489
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/59952489
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/59952489
Branch: refs/heads/release-0.8
Commit: 59952489b0d9036e5ff91374a85760005ee16d63
Parents: c6b90ee
Author: Gyula Fora <gy...@apache.org>
Authored: Tue Dec 23 21:16:46 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:04:17 2015 +0100
----------------------------------------------------------------------
.../streaming/api/datastream/DataStream.java | 39 +++---
.../api/datastream/StreamJoinOperator.java | 28 +++--
.../api/function/co/JoinWindowFunction.java | 6 +-
.../streaming/util/keys/ArrayKeySelector.java | 45 -------
.../streaming/util/keys/FieldsKeySelector.java | 103 ---------------
.../streaming/util/keys/KeySelectorUtil.java | 126 +++++++++++++++++++
.../streaming/util/keys/ObjectKeySelector.java | 30 -----
.../streaming/util/keys/PojoKeySelector.java | 98 ---------------
.../streaming/util/keys/TupleKeySelector.java | 44 -------
.../streaming/api/AggregationFunctionTest.java | 39 +++---
.../streaming/api/WindowCrossJoinTest.java | 42 ++++---
.../operator/CoGroupedBatchReduceTest.java | 30 +++--
.../invokable/operator/CoGroupedReduceTest.java | 42 ++++++-
.../operator/CoGroupedWindowReduceTest.java | 36 ++++--
.../operator/GroupedReduceInvokableTest.java | 13 +-
.../operator/GroupedWindowInvokableTest.java | 50 ++++----
.../partitioner/FieldsPartitionerTest.java | 12 +-
.../streaming/util/FieldsKeySelectorTest.java | 61 ---------
.../scala/streaming/windowing/WindowJoin.scala | 1 -
.../scala/streaming/CaseClassKeySelector.scala | 45 -------
.../flink/api/scala/streaming/DataStream.scala | 18 +--
.../api/scala/streaming/FieldsKeySelector.scala | 38 ------
.../scala/streaming/StreamCrossOperator.scala | 7 +-
.../scala/streaming/StreamJoinOperator.scala | 30 +++--
.../scala/streaming/WindowedDataStream.scala | 9 +-
25 files changed, 377 insertions(+), 615 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index c4e3368..422df5b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -33,6 +33,7 @@ import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -66,8 +67,7 @@ import org.apache.flink.streaming.partitioner.DistributePartitioner;
import org.apache.flink.streaming.partitioner.FieldsPartitioner;
import org.apache.flink.streaming.partitioner.ShufflePartitioner;
import org.apache.flink.streaming.partitioner.StreamPartitioner;
-import org.apache.flink.streaming.util.keys.FieldsKeySelector;
-import org.apache.flink.streaming.util.keys.PojoKeySelector;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
/**
* A DataStream represents a stream of elements of the same type. A DataStream
@@ -245,9 +245,11 @@ public class DataStream<OUT> {
* @return The grouped {@link DataStream}
*/
public GroupedDataStream<OUT> groupBy(int... fields) {
-
- return groupBy(FieldsKeySelector.getSelector(getType(), fields));
-
+ if (getType() instanceof BasicArrayTypeInfo || getType() instanceof PrimitiveArrayTypeInfo) {
+ return groupBy(new KeySelectorUtil.ArrayKeySelector<OUT>(fields));
+ } else {
+ return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
+ }
}
/**
@@ -264,7 +266,7 @@ public class DataStream<OUT> {
**/
public GroupedDataStream<OUT> groupBy(String... fields) {
- return groupBy(new PojoKeySelector<OUT>(getType(), fields));
+ return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
}
@@ -282,6 +284,11 @@ public class DataStream<OUT> {
return new GroupedDataStream<OUT>(this, clean(keySelector));
}
+ private GroupedDataStream<OUT> groupBy(Keys<OUT> keys) {
+ return new GroupedDataStream<OUT>(this, clean(KeySelectorUtil.getSelectorForKeys(keys,
+ getType())));
+ }
+
/**
* Sets the partitioning of the {@link DataStream} so that the output is
* partitioned by the selected fields. This setting only effects the how the
@@ -293,9 +300,11 @@ public class DataStream<OUT> {
* @return The DataStream with fields partitioning set.
*/
public DataStream<OUT> partitionBy(int... fields) {
-
- return setConnectionType(new FieldsPartitioner<OUT>(FieldsKeySelector.getSelector(
- getType(), fields)));
+ if (getType() instanceof BasicArrayTypeInfo || getType() instanceof PrimitiveArrayTypeInfo) {
+ return partitionBy(new KeySelectorUtil.ArrayKeySelector<OUT>(fields));
+ } else {
+ return partitionBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
+ }
}
/**
@@ -309,9 +318,11 @@ public class DataStream<OUT> {
* @return The DataStream with fields partitioning set.
*/
public DataStream<OUT> partitionBy(String... fields) {
+ return partitionBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
+ }
- return setConnectionType(new FieldsPartitioner<OUT>(new PojoKeySelector<OUT>(getType(),
- fields)));
+ private DataStream<OUT> partitionBy(Keys<OUT> keys) {
+ return partitionBy(KeySelectorUtil.getSelectorForKeys(keys, getType()));
}
/**
@@ -411,7 +422,7 @@ public class DataStream<OUT> {
* the data stream that will be fed back and used as the input for the
* iteration head. A common usage pattern for streaming iterations is to use
* output splitting to send a part of the closing data stream to the head.
- * Refer to {@link SingleOutputStreamOperator#split(OutputSelector)} for
+ * Refer to {@link SingleOutputStreamOperator#split(outputSelector)} for
* more information.
* <p>
* The iteration edge will be partitioned the same way as the first input of
@@ -549,7 +560,7 @@ public class DataStream<OUT> {
* {@link StreamCrossOperator#onWindow} should be called to define the
* window.
* <p>
- * Call {@link StreamCrossOperator.CrossWindow#with(CrossFunction)} to
+ * Call {@link StreamCrossOperator.CrossWindow#with(crossFunction)} to
* define a custom cross function.
*
* @param dataStreamToCross
@@ -572,7 +583,7 @@ public class DataStream<OUT> {
* window, and then the {@link StreamJoinOperator.JoinWindow#where} and
* {@link StreamJoinOperator.JoinPredicate#equalTo} can be used to define
* the join keys.</p> The user can also use the
- * {@link StreamJoinOperator.JoinedStream#with(JoinFunction)} to apply
+ * {@link StreamJoinOperator.JoinedStream#with(joinFunction)} to apply
* custom join function.
*
* @param other
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
index cbcc1b4..de15515 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
@@ -21,14 +21,14 @@ package org.apache.flink.streaming.api.datastream;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.util.keys.FieldsKeySelector;
-import org.apache.flink.streaming.util.keys.PojoKeySelector;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
public class StreamJoinOperator<I1, I2> extends
TemporalOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
@@ -45,9 +45,11 @@ public class StreamJoinOperator<I1, I2> extends
public static class JoinWindow<I1, I2> {
private StreamJoinOperator<I1, I2> op;
+ private TypeInformation<I1> type1;
private JoinWindow(StreamJoinOperator<I1, I2> operator) {
this.op = operator;
+ this.type1 = op.input1.getType();
}
/**
@@ -64,8 +66,8 @@ public class StreamJoinOperator<I1, I2> extends
* {@link JoinPredicate#equalTo} to continue the Join.
*/
public JoinPredicate<I1, I2> where(int... fields) {
- return new JoinPredicate<I1, I2>(op, FieldsKeySelector.getSelector(op.input1.getType(),
- fields));
+ return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<I1>(fields, type1), type1));
}
/**
@@ -81,8 +83,8 @@ public class StreamJoinOperator<I1, I2> extends
* {@link JoinPredicate#equalTo} to continue the Join.
*/
public JoinPredicate<I1, I2> where(String... fields) {
- return new JoinPredicate<I1, I2>(op, new PojoKeySelector<I1>(op.input1.getType(),
- fields));
+ return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<I1>(fields, type1), type1));
}
/**
@@ -114,13 +116,15 @@ public class StreamJoinOperator<I1, I2> extends
*/
public static class JoinPredicate<I1, I2> {
- public StreamJoinOperator<I1, I2> op;
- public KeySelector<I1, ?> keys1;
- public KeySelector<I2, ?> keys2;
+ private StreamJoinOperator<I1, I2> op;
+ private KeySelector<I1, ?> keys1;
+ private KeySelector<I2, ?> keys2;
+ private TypeInformation<I2> type2;
private JoinPredicate(StreamJoinOperator<I1, I2> operator, KeySelector<I1, ?> keys1) {
this.op = operator;
this.keys1 = keys1;
+ this.type2 = op.input2.getType();
}
/**
@@ -138,7 +142,8 @@ public class StreamJoinOperator<I1, I2> extends
* apply a custom wrapping
*/
public JoinedStream<I1, I2> equalTo(int... fields) {
- keys2 = FieldsKeySelector.getSelector(op.input2.getType(), fields);
+ keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields, type2),
+ type2);
return createJoinOperator();
}
@@ -156,7 +161,8 @@ public class StreamJoinOperator<I1, I2> extends
* apply a custom wrapping
*/
public JoinedStream<I1, I2> equalTo(String... fields) {
- this.keys2 = new PojoKeySelector<I2>(op.input2.getType(), fields);
+ this.keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields,
+ type2), type2);
return createJoinOperator();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
index 9f5cd5d..9b39f33 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
@@ -41,8 +41,12 @@ public class JoinWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1,
@Override
public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
for (IN1 item1 : first) {
+ Object key1 = keySelector1.getKey(item1);
+
for (IN2 item2 : second) {
- if (keySelector1.getKey(item1).equals(keySelector2.getKey(item2))) {
+ Object key2 = keySelector2.getKey(item2);
+
+ if (key1.equals(key2)) {
out.collect(joinFunction.join(item1, item2));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ArrayKeySelector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ArrayKeySelector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ArrayKeySelector.java
deleted file mode 100644
index 6d6f620..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ArrayKeySelector.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util.keys;
-
-import java.lang.reflect.Array;
-
-import org.apache.flink.api.java.tuple.Tuple;
-
-public class ArrayKeySelector<IN> extends FieldsKeySelector<IN> {
-
- private static final long serialVersionUID = 1L;
-
- public ArrayKeySelector(int... fields) {
- super(fields);
- }
-
- @Override
- public Object getKey(IN value) throws Exception {
- if (simpleKey) {
- return Array.get(value, keyFields[0]);
- } else {
- int c = 0;
- for (int pos : keyFields) {
- ((Tuple) key).setField(Array.get(value, pos), c);
- c++;
- }
- return key;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java
deleted file mode 100644
index 171ddc9..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util.keys;
-
-import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple10;
-import org.apache.flink.api.java.tuple.Tuple11;
-import org.apache.flink.api.java.tuple.Tuple12;
-import org.apache.flink.api.java.tuple.Tuple13;
-import org.apache.flink.api.java.tuple.Tuple14;
-import org.apache.flink.api.java.tuple.Tuple15;
-import org.apache.flink.api.java.tuple.Tuple16;
-import org.apache.flink.api.java.tuple.Tuple17;
-import org.apache.flink.api.java.tuple.Tuple18;
-import org.apache.flink.api.java.tuple.Tuple19;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple20;
-import org.apache.flink.api.java.tuple.Tuple21;
-import org.apache.flink.api.java.tuple.Tuple22;
-import org.apache.flink.api.java.tuple.Tuple23;
-import org.apache.flink.api.java.tuple.Tuple24;
-import org.apache.flink.api.java.tuple.Tuple25;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.api.java.tuple.Tuple5;
-import org.apache.flink.api.java.tuple.Tuple6;
-import org.apache.flink.api.java.tuple.Tuple7;
-import org.apache.flink.api.java.tuple.Tuple8;
-import org.apache.flink.api.java.tuple.Tuple9;
-
-public abstract class FieldsKeySelector<IN> implements KeySelector<IN, Object> {
-
- private static final long serialVersionUID = 1L;
-
- protected int[] keyFields;
- protected Object key;
- protected boolean simpleKey;
-
- @SuppressWarnings("unchecked")
- public static Class<? extends Tuple>[] tupleClasses = new Class[] { Tuple1.class, Tuple2.class,
- Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
- Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
- Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
- Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
- Tuple24.class, Tuple25.class };
-
- public FieldsKeySelector(int... fields) {
- this.keyFields = fields;
- this.simpleKey = fields.length == 1;
- for (int i : fields) {
- if (i < 0) {
- throw new RuntimeException("Grouping fields must be non-negative");
- }
- }
-
- try {
- key = tupleClasses[fields.length - 1].newInstance();
- } catch (Exception e) {
- throw new RuntimeException(e.getMessage());
- }
-
- }
-
- public static <R> KeySelector<R, ?> getSelector(TypeInformation<R> type, int... fields) {
- if (type.isTupleType()) {
- return new TupleKeySelector<R>(fields);
- } else if (type instanceof BasicArrayTypeInfo || type instanceof PrimitiveArrayTypeInfo) {
- return new ArrayKeySelector<R>(fields);
- } else {
- if (fields.length > 1) {
- throw new RuntimeException(
- "For non-tuple types use single field 0 or KeyExctractor for grouping");
-
- } else if (fields[0] > 0) {
- throw new RuntimeException(
- "For simple objects grouping only allowed on the first field");
- } else {
- return new ObjectKeySelector<R>();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
new file mode 100644
index 0000000..73e6360
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/KeySelectorUtil.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.util.keys;
+
+import java.lang.reflect.Array;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.common.typeutils.TypeComparator;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+
+public class KeySelectorUtil {
+
+ public static Class<?>[] tupleClasses = new Class[] { Tuple1.class, Tuple2.class, Tuple3.class,
+ Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class, Tuple9.class,
+ Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class, Tuple14.class,
+ Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class, Tuple19.class,
+ Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class, Tuple24.class,
+ Tuple25.class };
+
+ public static <X> KeySelector<X, ?> getSelectorForKeys(Keys<X> keys, TypeInformation<X> typeInfo) {
+ int[] logicalKeyPositions = keys.computeLogicalKeyPositions();
+ int keyLength = logicalKeyPositions.length;
+ boolean[] orders = new boolean[keyLength];
+ TypeComparator<X> comparator = ((CompositeType<X>) typeInfo).createComparator(
+ logicalKeyPositions, orders, 0);
+ return new ComparableKeySelector<X>(comparator, keyLength);
+ }
+
+ public static class ComparableKeySelector<IN> implements KeySelector<IN, Tuple> {
+
+ private static final long serialVersionUID = 1L;
+
+ private TypeComparator<IN> comparator;
+ private int keyLength;
+ private Object[] keyArray;
+ private Tuple key;
+
+ public ComparableKeySelector(TypeComparator<IN> comparator, int keyLength) {
+ this.comparator = comparator;
+ this.keyLength = keyLength;
+ keyArray = new Object[keyLength];
+ try {
+ key = (Tuple) tupleClasses[keyLength - 1].newInstance();
+ } catch (Exception e) {
+ }
+ }
+
+ @Override
+ public Tuple getKey(IN value) throws Exception {
+ comparator.extractKeys(value, keyArray, 0);
+ for (int i = 0; i < keyLength; i++) {
+ key.setField(keyArray[i], i);
+ }
+ return key;
+ }
+
+ }
+
+ public static class ArrayKeySelector<IN> implements KeySelector<IN, Tuple> {
+
+ private static final long serialVersionUID = 1L;
+
+ Tuple key;
+ int[] fields;
+
+ public ArrayKeySelector(int... fields) {
+ this.fields = fields;
+ try {
+ key = (Tuple) tupleClasses[fields.length - 1].newInstance();
+ } catch (Exception e) {
+ }
+ }
+
+ @Override
+ public Tuple getKey(IN value) throws Exception {
+ for (int i = 0; i < fields.length; i++) {
+ int pos = fields[i];
+ key.setField(Array.get(value, fields[pos]), i);
+ }
+ return key;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ObjectKeySelector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ObjectKeySelector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ObjectKeySelector.java
deleted file mode 100644
index cf4ecc6..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/ObjectKeySelector.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util.keys;
-
-import org.apache.flink.api.java.functions.KeySelector;
-
-public class ObjectKeySelector<IN> implements KeySelector<IN, IN> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public IN getKey(IN value) throws Exception {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/PojoKeySelector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/PojoKeySelector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/PojoKeySelector.java
deleted file mode 100644
index 618f14f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/PojoKeySelector.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util.keys;
-
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.CompositeType;
-import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.typeutils.PojoTypeInfo;
-import org.apache.flink.api.java.typeutils.runtime.PojoComparator;
-
-public class PojoKeySelector<IN> extends FieldsKeySelector<IN> {
-
- private static final long serialVersionUID = 1L;
-
- PojoComparator<IN> comparator;
-
- public PojoKeySelector(TypeInformation<IN> type, String... fields) {
- super(new int[removeDuplicates(fields).length]);
- if (!(type instanceof CompositeType<?>)) {
- throw new IllegalArgumentException(
- "Key expressions are only supported on POJO types and Tuples. "
- + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
- }
- CompositeType<IN> cType = (CompositeType<IN>) type;
-
- String[] keyFields = removeDuplicates(fields);
- int numOfKeys = keyFields.length;
-
- List<FlatFieldDescriptor> fieldDescriptors = new ArrayList<FlatFieldDescriptor>();
- for (String field : keyFields) {
- cType.getKey(field, 0, fieldDescriptors);
- }
-
- int[] logicalKeyPositions = new int[numOfKeys];
- boolean[] orders = new boolean[numOfKeys];
-
- for (int i = 0; i < numOfKeys; i++) {
- logicalKeyPositions[i] = fieldDescriptors.get(i).getPosition();
- }
-
- if (cType instanceof PojoTypeInfo) {
- comparator = (PojoComparator<IN>) cType
- .createComparator(logicalKeyPositions, orders, 0);
- } else {
- throw new IllegalArgumentException(
- "Key expressions are only supported on POJO types. "
- + "A type is considered a POJO if all its fields are public, or have both getters and setters defined");
- }
-
- }
-
- @Override
- public Object getKey(IN value) throws Exception {
-
- Field[] keyFields = comparator.getKeyFields();
- if (simpleKey) {
- return comparator.accessField(keyFields[0], value);
- } else {
- int c = 0;
- for (Field field : keyFields) {
- ((Tuple) key).setField(comparator.accessField(field, value), c);
- c++;
- }
- }
- return key;
- }
-
- private static String[] removeDuplicates(String[] in) {
- List<String> ret = new LinkedList<String>();
- for (String el : in) {
- if (!ret.contains(el)) {
- ret.add(el);
- }
- }
- return ret.toArray(new String[ret.size()]);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/TupleKeySelector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/TupleKeySelector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/TupleKeySelector.java
deleted file mode 100644
index 4ca64ef..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/TupleKeySelector.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util.keys;
-
-import org.apache.flink.api.java.tuple.Tuple;
-
-public class TupleKeySelector<IN> extends FieldsKeySelector<IN> {
-
- private static final long serialVersionUID = 1L;
-
- public TupleKeySelector(int... fields) {
- super(fields);
- }
-
- @Override
- public Object getKey(IN value) throws Exception {
- if (simpleKey) {
- return ((Tuple) value).getField(keyFields[0]);
- } else {
- int c = 0;
- for (int pos : keyFields) {
- ((Tuple) key).setField(((Tuple) value).getField(pos), c);
- c++;
- }
- return key;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
index 9376166..49cd497 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/AggregationFunctionTest.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -34,7 +36,7 @@ import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable;
import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable;
import org.apache.flink.streaming.util.MockContext;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
import org.junit.Test;
public class AggregationFunctionTest {
@@ -94,14 +96,14 @@ public class AggregationFunctionTest {
Integer.class, type1);
ReduceFunction<Integer> sumFunction0 = SumAggregator
.getSumFunction(0, Integer.class, type2);
- ReduceFunction<Tuple2<Integer, Integer>> minFunction = ComparableAggregator
- .getAggregator(1, type1, AggregationType.MIN);
- ReduceFunction<Integer> minFunction0 = ComparableAggregator.getAggregator(0,
- type2, AggregationType.MIN);
- ReduceFunction<Tuple2<Integer, Integer>> maxFunction = ComparableAggregator
- .getAggregator(1, type1, AggregationType.MAX);
- ReduceFunction<Integer> maxFunction0 = ComparableAggregator.getAggregator(0,
- type2, AggregationType.MAX);
+ ReduceFunction<Tuple2<Integer, Integer>> minFunction = ComparableAggregator.getAggregator(
+ 1, type1, AggregationType.MIN);
+ ReduceFunction<Integer> minFunction0 = ComparableAggregator.getAggregator(0, type2,
+ AggregationType.MIN);
+ ReduceFunction<Tuple2<Integer, Integer>> maxFunction = ComparableAggregator.getAggregator(
+ 1, type1, AggregationType.MAX);
+ ReduceFunction<Integer> maxFunction0 = ComparableAggregator.getAggregator(0, type2,
+ AggregationType.MAX);
List<Tuple2<Integer, Integer>> sumList = MockContext.createAndExecute(
new StreamReduceInvokable<Tuple2<Integer, Integer>>(sumFunction), getInputList());
@@ -111,17 +113,24 @@ public class AggregationFunctionTest {
List<Tuple2<Integer, Integer>> maxList = MockContext.createAndExecute(
new StreamReduceInvokable<Tuple2<Integer, Integer>>(maxFunction), getInputList());
+ TypeInformation<Tuple2<Integer, Integer>> typeInfo = TypeExtractor
+ .getForObject(new Tuple2<Integer, Integer>(1, 1));
+
+ KeySelector<Tuple2<Integer, Integer>, ?> keySelector = KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<Tuple2<Integer, Integer>>(new int[] { 0 }, typeInfo),
+ typeInfo);
+
List<Tuple2<Integer, Integer>> groupedSumList = MockContext.createAndExecute(
- new GroupedReduceInvokable<Tuple2<Integer, Integer>>(sumFunction,
- new TupleKeySelector<Tuple2<Integer, Integer>>(0)), getInputList());
+ new GroupedReduceInvokable<Tuple2<Integer, Integer>>(sumFunction, keySelector),
+ getInputList());
List<Tuple2<Integer, Integer>> groupedMinList = MockContext.createAndExecute(
- new GroupedReduceInvokable<Tuple2<Integer, Integer>>(minFunction,
- new TupleKeySelector<Tuple2<Integer, Integer>>(0)), getInputList());
+ new GroupedReduceInvokable<Tuple2<Integer, Integer>>(minFunction, keySelector),
+ getInputList());
List<Tuple2<Integer, Integer>> groupedMaxList = MockContext.createAndExecute(
- new GroupedReduceInvokable<Tuple2<Integer, Integer>>(maxFunction,
- new TupleKeySelector<Tuple2<Integer, Integer>>(0)), getInputList());
+ new GroupedReduceInvokable<Tuple2<Integer, Integer>>(maxFunction, keySelector),
+ getInputList());
assertEquals(expectedSumList, sumList);
assertEquals(expectedMinList, minList);
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
index d8cdfa5..b71bb25 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -23,6 +23,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
@@ -49,16 +50,16 @@ public class WindowCrossJoinTest implements Serializable {
env.setBufferTimeout(1);
ArrayList<Tuple2<Integer, String>> in1 = new ArrayList<Tuple2<Integer, String>>();
- ArrayList<Integer> in2 = new ArrayList<Integer>();
+ ArrayList<Tuple1<Integer>> in2 = new ArrayList<Tuple1<Integer>>();
in1.add(new Tuple2<Integer, String>(10, "a"));
in1.add(new Tuple2<Integer, String>(20, "b"));
in1.add(new Tuple2<Integer, String>(20, "x"));
in1.add(new Tuple2<Integer, String>(0, "y"));
- in2.add(0);
- in2.add(5);
- in2.add(20);
+ in2.add(new Tuple1<Integer>(0));
+ in2.add(new Tuple1<Integer>(5));
+ in2.add(new Tuple1<Integer>(20));
joinExpectedResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(
new Tuple2<Integer, String>(20, "b"), 20));
@@ -93,23 +94,24 @@ public class WindowCrossJoinTest implements Serializable {
new Tuple2<Integer, String>(0, "y"), 20));
DataStream<Tuple2<Integer, String>> inStream1 = env.fromCollection(in1);
- DataStream<Integer> inStream2 = env.fromCollection(in2);
+ DataStream<Tuple1<Integer>> inStream2 = env.fromCollection(in2);
inStream1.join(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
.where(0).equalTo(0).addSink(new JoinResultSink());
- inStream1.cross(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
- .with(new CrossFunction<Tuple2<Integer,String>, Integer, Tuple2<Tuple2<Integer,String>, Integer>>() {
+ inStream1
+ .cross(inStream2)
+ .onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
+ .with(new CrossFunction<Tuple2<Integer, String>, Tuple1<Integer>, Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>>() {
private static final long serialVersionUID = 1L;
@Override
- public Tuple2<Tuple2<Integer, String>, Integer> cross(
- Tuple2<Integer, String> val1, Integer val2) throws Exception {
- return new Tuple2<Tuple2<Integer,String>, Integer>(val1, val2);
+ public Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> cross(
+ Tuple2<Integer, String> val1, Tuple1<Integer> val2) throws Exception {
+ return new Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>(val1, val2);
}
- })
- .addSink(new CrossResultSink());
+ }).addSink(new CrossResultSink());
env.executeTest(MEMORYSIZE);
@@ -131,11 +133,11 @@ public class WindowCrossJoinTest implements Serializable {
}
}
- private static class MyTimestamp2 implements TimeStamp<Integer> {
+ private static class MyTimestamp2 implements TimeStamp<Tuple1<Integer>> {
private static final long serialVersionUID = 1L;
@Override
- public long getTimestamp(Integer value) {
+ public long getTimestamp(Tuple1<Integer> value) {
return 101L;
}
@@ -146,22 +148,22 @@ public class WindowCrossJoinTest implements Serializable {
}
private static class JoinResultSink implements
- SinkFunction<Tuple2<Tuple2<Integer, String>, Integer>> {
+ SinkFunction<Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>> {
private static final long serialVersionUID = 1L;
@Override
- public void invoke(Tuple2<Tuple2<Integer, String>, Integer> value) {
- joinResults.add(value);
+ public void invoke(Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> value) {
+ joinResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(value.f0, value.f1.f0));
}
}
private static class CrossResultSink implements
- SinkFunction<Tuple2<Tuple2<Integer, String>, Integer>> {
+ SinkFunction<Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>> {
private static final long serialVersionUID = 1L;
@Override
- public void invoke(Tuple2<Tuple2<Integer, String>, Integer> value) {
- crossResults.add(value);
+ public void invoke(Tuple2<Tuple2<Integer, String>, Tuple1<Integer>> value) {
+ crossResults.add(new Tuple2<Tuple2<Integer, String>, Integer>(value.f0, value.f1.f0));
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java
index bc19a89..1d0732c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java
@@ -23,15 +23,35 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedBatchReduceInvokable;
import org.apache.flink.streaming.util.MockCoContext;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
import org.junit.Test;
public class CoGroupedBatchReduceTest {
+ KeySelector<Tuple2<String, String>, ?> keySelector1 = new KeySelector<Tuple2<String, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple2<String, String> value) throws Exception {
+ return value.f0;
+ }
+ };
+
+ KeySelector<Tuple2<String, Integer>, ?> keySelector2 = new KeySelector<Tuple2<String, Integer>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple2<String, Integer> value) throws Exception {
+ return value.f0;
+ }
+ };
+
private static class MyCoReduceFunction implements
CoReduceFunction<Tuple2<String, Integer>, Tuple2<String, String>, String> {
private static final long serialVersionUID = 1L;
@@ -59,7 +79,6 @@ public class CoGroupedBatchReduceTest {
}
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void coGroupedBatchReduceTest1() {
@@ -96,8 +115,7 @@ public class CoGroupedBatchReduceTest {
expected.add("h");
CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 4L, 3L, new TupleKeySelector(0),
- new TupleKeySelector(0));
+ new MyCoReduceFunction(), 4L, 3L, 4L, 3L, keySelector2, keySelector1);
List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
@@ -106,7 +124,6 @@ public class CoGroupedBatchReduceTest {
assertEquals(expected, result);
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void coGroupedBatchReduceTest2() {
@@ -143,8 +160,7 @@ public class CoGroupedBatchReduceTest {
expected.add("fh");
CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 2L, 2L, new TupleKeySelector(0),
- new TupleKeySelector(0));
+ new MyCoReduceFunction(), 4L, 3L, 2L, 2L, keySelector2, keySelector1);
List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
index 15d42a4..273bbae 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
@@ -22,12 +22,12 @@ import static org.junit.Assert.assertEquals;
import java.util.Arrays;
import java.util.List;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedReduceInvokable;
import org.apache.flink.streaming.util.MockCoContext;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
import org.junit.Test;
public class CoGroupedReduceTest {
@@ -59,7 +59,7 @@ public class CoGroupedReduceTest {
}
}
- @SuppressWarnings({ "unchecked", "rawtypes" })
+ @SuppressWarnings("unchecked")
@Test
public void coGroupedReduceTest() {
Tuple3<String, String, String> word1 = new Tuple3<String, String, String>("a", "word1", "b");
@@ -71,8 +71,38 @@ public class CoGroupedReduceTest {
Tuple2<Integer, Integer> int4 = new Tuple2<Integer, Integer>(2, 4);
Tuple2<Integer, Integer> int5 = new Tuple2<Integer, Integer>(1, 5);
+ KeySelector<Tuple3<String, String, String>, ?> keySelector0 = new KeySelector<Tuple3<String, String, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple3<String, String, String> value) throws Exception {
+ return value.f0;
+ }
+ };
+
+ KeySelector<Tuple2<Integer, Integer>, ?> keySelector1 = new KeySelector<Tuple2<Integer, Integer>, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
+ return value.f0;
+ }
+ };
+
+ KeySelector<Tuple3<String, String, String>, ?> keySelector2 = new KeySelector<Tuple3<String, String, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple3<String, String, String> value) throws Exception {
+ return value.f2;
+ }
+ };
+
CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
- new MyCoReduceFunction(), new TupleKeySelector(0), new TupleKeySelector(0));
+ new MyCoReduceFunction(), keySelector0, keySelector1);
List<String> expected = Arrays.asList("word1", "1", "word2", "2", "word1word3", "3", "5",
"7");
@@ -83,12 +113,12 @@ public class CoGroupedReduceTest {
assertEquals(expected, actualList);
invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
- new MyCoReduceFunction(), new TupleKeySelector(2), new TupleKeySelector(0));
+ new MyCoReduceFunction(), keySelector2, keySelector1);
expected = Arrays.asList("word1", "1", "word2", "2", "word2word3", "3", "5", "7");
- actualList = MockCoContext.createAndExecute(invokable,
- Arrays.asList(word1, word2, word3), Arrays.asList(int1, int2, int3, int4, int5));
+ actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(word1, word2, word3),
+ Arrays.asList(int1, int2, int3, int4, int5));
assertEquals(expected, actualList);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
index b5a7e8d..e3f2a1b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
@@ -25,16 +25,36 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedWindowReduceInvokable;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.util.MockCoContext;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
import org.junit.Test;
public class CoGroupedWindowReduceTest {
+ KeySelector<Tuple2<String, Integer>, ?> keySelector0 = new KeySelector<Tuple2<String, Integer>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple2<String, Integer> value) throws Exception {
+ return value.f0;
+ }
+ };
+
+ KeySelector<Tuple2<String, String>, ?> keySelector1 = new KeySelector<Tuple2<String, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple2<String, String> value) throws Exception {
+ return value.f0;
+ }
+ };
+
private static class MyCoReduceFunction implements
CoReduceFunction<Tuple2<String, Integer>, Tuple2<String, String>, String> {
private static final long serialVersionUID = 1L;
@@ -85,7 +105,6 @@ public class CoGroupedWindowReduceTest {
}
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void coGroupedWindowReduceTest1() {
@@ -125,9 +144,9 @@ public class CoGroupedWindowReduceTest {
expected.add("i");
CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 4L, 3L, new TupleKeySelector(0),
- new TupleKeySelector( 0), new MyTimeStamp<Tuple2<String, Integer>>(
- timestamps1), new MyTimeStamp<Tuple2<String, String>>(timestamps2));
+ new MyCoReduceFunction(), 4L, 3L, 4L, 3L, keySelector0, keySelector1,
+ new MyTimeStamp<Tuple2<String, Integer>>(timestamps1),
+ new MyTimeStamp<Tuple2<String, String>>(timestamps2));
List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
@@ -136,7 +155,6 @@ public class CoGroupedWindowReduceTest {
assertEquals(expected, result);
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void coGroupedWindowReduceTest2() {
@@ -178,9 +196,9 @@ public class CoGroupedWindowReduceTest {
expected.add("fh");
CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 2L, 2L, new TupleKeySelector( 0),
- new TupleKeySelector( 0), new MyTimeStamp<Tuple2<String, Integer>>(
- timestamps1), new MyTimeStamp<Tuple2<String, String>>(timestamps2));
+ new MyCoReduceFunction(), 4L, 3L, 2L, 2L, keySelector0, keySelector1,
+ new MyTimeStamp<Tuple2<String, Integer>>(timestamps1),
+ new MyTimeStamp<Tuple2<String, String>>(timestamps2));
List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
index ceaccf3..ce47c67 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedReduceInvokableTest.java
@@ -23,8 +23,8 @@ import java.util.Arrays;
import java.util.List;
import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.util.MockContext;
-import org.apache.flink.streaming.util.keys.ObjectKeySelector;
import org.junit.Test;
public class GroupedReduceInvokableTest {
@@ -43,7 +43,15 @@ public class GroupedReduceInvokableTest {
@Test
public void test() {
GroupedReduceInvokable<Integer> invokable1 = new GroupedReduceInvokable<Integer>(
- new MyReducer(), new ObjectKeySelector<Integer>());
+ new MyReducer(), new KeySelector<Integer, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) throws Exception {
+ return value;
+ }
+ });
List<Integer> expected = Arrays.asList(1, 2, 2, 4, 3);
List<Integer> actual = MockContext.createAndExecute(invokable1,
@@ -51,5 +59,4 @@ public class GroupedReduceInvokableTest {
assertEquals(expected, actual);
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
index c3a48d5..d97cadc 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
@@ -40,11 +40,20 @@ import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
import org.apache.flink.streaming.util.MockContext;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
import org.junit.Test;
public class GroupedWindowInvokableTest {
+ KeySelector<Tuple2<Integer, String>, ?> keySelector = new KeySelector<Tuple2<Integer, String>, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple2<Integer, String> value) throws Exception {
+ return value.f1;
+ }
+ };
+
/**
* Tests that illegal arguments result in failure. The following cases are
* tested: 1) having no trigger 2) having no eviction 3) having neither
@@ -162,7 +171,7 @@ public class GroupedWindowInvokableTest {
expectedDistributedEviction.add(3);
expectedDistributedEviction.add(3);
expectedDistributedEviction.add(15);
-
+
List<Integer> expectedCentralEviction = new ArrayList<Integer>();
expectedCentralEviction.add(2);
expectedCentralEviction.add(5);
@@ -173,7 +182,7 @@ public class GroupedWindowInvokableTest {
expectedCentralEviction.add(5);
expectedCentralEviction.add(1);
expectedCentralEviction.add(5);
-
+
LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
// Trigger on every 2nd element, but the first time after the 3rd
triggers.add(new CountTriggerPolicy<Integer>(2, -1));
@@ -185,7 +194,7 @@ public class GroupedWindowInvokableTest {
LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
- ReduceFunction<Integer> reduceFunction=new ReduceFunction<Integer>() {
+ ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
private static final long serialVersionUID = 1L;
@Override
@@ -193,8 +202,8 @@ public class GroupedWindowInvokableTest {
return value1 + value2;
}
};
-
- KeySelector<Integer, Integer> keySelector=new KeySelector<Integer, Integer>() {
+
+ KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
private static final long serialVersionUID = 1L;
@Override
@@ -202,7 +211,7 @@ public class GroupedWindowInvokableTest {
return value;
}
};
-
+
GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
reduceFunction, keySelector, triggers, evictions, centralTriggers, null);
@@ -213,18 +222,19 @@ public class GroupedWindowInvokableTest {
actual.add(current);
}
- assertEquals(new HashSet<Integer>(expectedDistributedEviction), new HashSet<Integer>(actual));
+ assertEquals(new HashSet<Integer>(expectedDistributedEviction),
+ new HashSet<Integer>(actual));
assertEquals(expectedDistributedEviction.size(), actual.size());
-
- //Run test with central eviction
+
+ // Run test with central eviction
triggers.clear();
centralTriggers.add(new CountTriggerPolicy<Integer>(2, -1));
LinkedList<EvictionPolicy<Integer>> centralEvictions = new LinkedList<EvictionPolicy<Integer>>();
centralEvictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
- invokable = new GroupedWindowInvokable<Integer, Integer>(
- reduceFunction, keySelector, triggers, null, centralTriggers,centralEvictions);
-
+
+ invokable = new GroupedWindowInvokable<Integer, Integer>(reduceFunction, keySelector,
+ triggers, null, centralTriggers, centralEvictions);
+
result = MockContext.createAndExecute(invokable, inputs);
actual = new LinkedList<Integer>();
for (Integer current : result) {
@@ -279,8 +289,7 @@ public class GroupedWindowInvokableTest {
return value2;
}
}
- }, new TupleKeySelector<Tuple2<Integer, String>>(1), triggers, evictions,
- centralTriggers, null);
+ }, keySelector, triggers, evictions, centralTriggers, null);
List<Tuple2<Integer, String>> result = MockContext.createAndExecute(invokable2, inputs2);
@@ -387,8 +396,7 @@ public class GroupedWindowInvokableTest {
LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
- myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
- distributedTriggers, evictions, triggers, null);
+ myReduceFunction, keySelector, distributedTriggers, evictions, triggers, null);
ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
@@ -398,7 +406,7 @@ public class GroupedWindowInvokableTest {
assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
new HashSet<Tuple2<Integer, String>>(result));
assertEquals(expected.size(), result.size());
-
+
// repeat the test with central eviction. The result should be the same.
triggers.clear();
triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L));
@@ -407,8 +415,8 @@ public class GroupedWindowInvokableTest {
centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStamp));
invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
- myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
- distributedTriggers, evictions, triggers, centralEvictions);
+ myReduceFunction, keySelector, distributedTriggers, evictions, triggers,
+ centralEvictions);
result = new ArrayList<Tuple2<Integer, String>>();
for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
index 18b3015..b56649b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/FieldsPartitionerTest.java
@@ -20,11 +20,11 @@ package org.apache.flink.streaming.partitioner;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +42,15 @@ public class FieldsPartitionerTest {
@Before
public void setPartitioner() {
- fieldsPartitioner = new FieldsPartitioner<Tuple>(new TupleKeySelector<Tuple>(0));
+ fieldsPartitioner = new FieldsPartitioner<Tuple>(new KeySelector<Tuple, String>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String getKey(Tuple value) throws Exception {
+ return value.getField(0);
+ }
+ });
}
@Test
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/FieldsKeySelectorTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/FieldsKeySelectorTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/FieldsKeySelectorTest.java
deleted file mode 100644
index 98b60b5..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/FieldsKeySelectorTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.util;
-
-import static org.junit.Assert.assertEquals;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.util.keys.FieldsKeySelector;
-import org.apache.flink.streaming.util.keys.ObjectKeySelector;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
-import org.junit.Test;
-
-public class FieldsKeySelectorTest {
-
- @Test
- public void testGetKey() throws Exception {
-
- Integer i = 5;
- Tuple2<Integer, String> t = new Tuple2<Integer, String>(-1, "a");
- double[] a = new double[] { 0.0, 1.2 };
-
- KeySelector<Integer, ?> ks1 = new ObjectKeySelector<Integer>();
-
- assertEquals(ks1.getKey(i), 5);
-
- KeySelector<Tuple2<Integer, String>, ?> ks3 = new TupleKeySelector<Tuple2<Integer, String>>(
- 1);
- assertEquals(ks3.getKey(t), "a");
-
- KeySelector<Tuple2<Integer, String>, ?> ks4 = FieldsKeySelector.getSelector(
- TypeExtractor.getForObject(t), 1, 1);
- assertEquals(ks4.getKey(t), new Tuple2<String, String>("a", "a"));
-
- KeySelector<double[], ?> ks5 = FieldsKeySelector.getSelector(
- TypeExtractor.getForObject(a), 0);
- assertEquals(ks5.getKey(a), 0.0);
-
- KeySelector<double[], ?> ks6 = FieldsKeySelector.getSelector(
- TypeExtractor.getForObject(a), 1, 0);
- assertEquals(ks6.getKey(a), new Tuple2<Double, Double>(1.2, 0.0));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
index eea76c1..caf5eb9 100644
--- a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
+++ b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
@@ -20,7 +20,6 @@ package org.apache.flink.examples.scala.streaming.windowing
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment
-import org.apache.flink.streaming.api.function.source.SourceFunction
import org.apache.flink.util.Collector
import scala.util.Random
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala
deleted file mode 100644
index 63410a9..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.api.scala.streaming
-
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import java.util.ArrayList
-import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor
-import org.apache.flink.api.java.functions.KeySelector
-
-class CaseClassKeySelector[T <: Product](@transient val typeInfo: CaseClassTypeInfo[T],
- val keyFields: String*) extends KeySelector[T, Seq[Any]] {
-
- val numOfKeys: Int = keyFields.length;
-
- @transient val fieldDescriptors = new ArrayList[FlatFieldDescriptor]();
- for (field <- keyFields) {
- typeInfo.getKey(field, 0, fieldDescriptors);
- }
-
- val logicalKeyPositions = new Array[Int](numOfKeys)
- val orders = new Array[Boolean](numOfKeys)
-
- for (i <- 0 to numOfKeys - 1) {
- logicalKeyPositions(i) = fieldDescriptors.get(i).getPosition();
- }
-
- def getKey(value: T): Seq[Any] = {
- for (i <- 0 to numOfKeys - 1) yield value.productElement(logicalKeyPositions(i))
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 6df4b25..38ad384 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -116,7 +116,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def groupBy(fields: Int*): DataStream[T] =
- new DataStream[T](javaStream.groupBy(new FieldsKeySelector[T](fields: _*)))
+ new DataStream[T](javaStream.groupBy(fields: _*))
/**
* Groups the elements of a DataStream by the given field expressions to
@@ -124,12 +124,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def groupBy(firstField: String, otherFields: String*): DataStream[T] =
- javaStream.getType() match {
- case ccInfo: CaseClassTypeInfo[T] => new DataStream[T](javaStream.groupBy(
- new CaseClassKeySelector[T](ccInfo, firstField +: otherFields.toArray: _*)))
- case _ => new DataStream[T](javaStream.groupBy(
- firstField +: otherFields.toArray: _*))
- }
+ new DataStream[T](javaStream.groupBy(firstField +: otherFields.toArray: _*))
/**
* Groups the elements of a DataStream by the given K key to
@@ -152,7 +147,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def partitionBy(fields: Int*): DataStream[T] =
- new DataStream[T](javaStream.partitionBy(new FieldsKeySelector[T](fields: _*)))
+ new DataStream[T](javaStream.partitionBy(fields: _*));
/**
* Sets the partitioning of the DataStream so that the output is
@@ -161,12 +156,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
- javaStream.getType() match {
- case ccInfo: CaseClassTypeInfo[T] => new DataStream[T](javaStream.partitionBy(
- new CaseClassKeySelector[T](ccInfo, firstField +: otherFields.toArray: _*)))
- case _ => new DataStream[T](javaStream.partitionBy(
- firstField +: otherFields.toArray: _*))
- }
+ new DataStream[T](javaStream.partitionBy(firstField +: otherFields.toArray: _*))
/**
* Sets the partitioning of the DataStream so that the output is
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
deleted file mode 100644
index bc79fca..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-
-import org.apache.flink.streaming.util.keys.{ FieldsKeySelector => JavaSelector }
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.java.tuple.Tuple
-
-class FieldsKeySelector[IN](fields: Int*) extends KeySelector[IN, Seq[Any]] {
-
- override def getKey(value: IN): Seq[Any] =
-
- value match {
- case prod: Product =>
- for (i <- 0 to fields.length - 1) yield prod.productElement(fields(i))
- case tuple: Tuple =>
- for (i <- 0 to fields.length - 1) yield tuple.getField(fields(i))
-
- case _ => throw new RuntimeException("Only tuple types are supported")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
index 5f579e5..72052b9 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
@@ -23,17 +23,16 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.typeutils.TypeSerializer
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.scala.ClosureCleaner
-import org.apache.flink.api.scala.typeutils.CaseClassSerializer
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
import org.apache.flink.streaming.api.datastream.TemporalOperator
-import org.apache.flink.streaming.api.function.co.JoinWindowFunction
-import org.apache.flink.streaming.util.keys.PojoKeySelector
import scala.reflect.ClassTag
import org.apache.commons.lang.Validate
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.api.function.co.CrossWindowFunction
import org.apache.flink.api.common.functions.CrossFunction
+import org.apache.flink.api.scala.typeutils.CaseClassSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+
class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
index 4095645..f47d79e 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
@@ -28,10 +28,11 @@ import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
import org.apache.flink.streaming.api.datastream.TemporalOperator
import org.apache.flink.streaming.api.function.co.JoinWindowFunction
-import org.apache.flink.streaming.util.keys.PojoKeySelector
import scala.reflect.ClassTag
import org.apache.commons.lang.Validate
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
+import org.apache.flink.streaming.util.keys.KeySelectorUtil
+import org.apache.flink.api.java.operators.Keys
class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
@@ -43,8 +44,10 @@ TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
object StreamJoinOperator {
- class JoinWindow[I1, I2](op: StreamJoinOperator[I1, I2]) {
+ class JoinWindow[I1, I2](private[flink] op: StreamJoinOperator[I1, I2]) {
+ private[flink] val type1 = op.input1.getType();
+
/**
* Continues a temporal Join transformation by defining
* the fields in the first stream to be used as keys for the join.
@@ -52,7 +55,8 @@ object StreamJoinOperator {
* to define the second key.
*/
def where(fields: Int*) = {
- new JoinPredicate[I1, I2](op, new FieldsKeySelector[I1](fields: _*))
+ new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(fields.toArray,type1),type1))
}
/**
@@ -62,12 +66,8 @@ object StreamJoinOperator {
* to define the second key.
*/
def where(firstField: String, otherFields: String*) =
- op.input1.getType() match {
- case ccInfo: CaseClassTypeInfo[I1] => new JoinPredicate[I1, I2](op,
- new CaseClassKeySelector[I1](ccInfo, firstField +: otherFields.toArray: _*))
- case _ => new JoinPredicate[I1, I2](op, new PojoKeySelector[I1](
- op.input1.getType(), (firstField +: otherFields): _*))
- }
+ new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray,type1),type1))
/**
* Continues a temporal Join transformation by defining
@@ -90,6 +90,7 @@ object StreamJoinOperator {
class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2],
private[flink] val keys1: KeySelector[I1, _]) {
private[flink] var keys2: KeySelector[I2, _] = null
+ private[flink] val type2 = op.input2.getType();
/**
* Creates a temporal join transformation by defining the second join key.
@@ -98,7 +99,8 @@ object StreamJoinOperator {
* To define a custom wrapping, use JoinedStream.apply(...)
*/
def equalTo(fields: Int*): JoinedStream[I1, I2] = {
- finish(new FieldsKeySelector[I2](fields: _*))
+ finish(KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(fields.toArray,type2),type2))
}
/**
@@ -108,12 +110,8 @@ object StreamJoinOperator {
* To define a custom wrapping, use JoinedStream.apply(...)
*/
def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
- op.input2.getType() match {
- case ccInfo: CaseClassTypeInfo[I2] => finish(
- new CaseClassKeySelector[I2](ccInfo, firstField +: otherFields.toArray: _*))
- case _ => finish(new PojoKeySelector[I2](op.input2.getType(),
- (firstField +: otherFields): _*))
- }
+ finish(KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray,type2),type2))
/**
* Creates a temporal join transformation by defining the second join key.
http://git-wip-us.apache.org/repos/asf/flink/blob/59952489/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
index 11f042d..5346c4c 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -66,7 +66,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
*
*/
def groupBy(fields: Int*): WindowedDataStream[T] =
- new WindowedDataStream[T](javaStream.groupBy(new FieldsKeySelector[T](fields: _*)))
+ new WindowedDataStream[T](javaStream.groupBy(fields: _*))
/**
* Groups the elements of the WindowedDataStream using the given
@@ -78,13 +78,8 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
*
*/
def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
- javaStream.getType() match {
- case ccInfo: CaseClassTypeInfo[T] => new WindowedDataStream[T](javaStream.groupBy(
- new CaseClassKeySelector[T](ccInfo, firstField +: otherFields.toArray: _*)))
- case _ => new WindowedDataStream[T](javaStream.groupBy(
+ new WindowedDataStream[T](javaStream.groupBy(
firstField +: otherFields.toArray: _*))
- }
-
/**
* Groups the elements of the WindowedDataStream using the given
[09/36] flink git commit: [scala] [streaming] Temporal join operator
added
Posted by mb...@apache.org.
[scala] [streaming] Temporal join operator added
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/739fdf0b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/739fdf0b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/739fdf0b
Branch: refs/heads/release-0.8
Commit: 739fdf0b76560f96b1b2dd9d9dbfba6eff1170bc
Parents: 652327b
Author: Gyula Fora <gy...@apache.org>
Authored: Sat Dec 20 18:12:19 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:56:49 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 9 +
.../api/datastream/ConnectedDataStream.java | 4 +-
.../streaming/api/datastream/DataStream.java | 15 +-
.../api/datastream/GroupedDataStream.java | 2 +-
.../datastream/SingleOutputStreamOperator.java | 10 +-
.../api/datastream/StreamJoinOperator.java | 118 +++++++-----
.../api/datastream/StreamProjection.java | 2 +-
.../streaming/util/keys/FieldsKeySelector.java | 15 +-
.../streaming/api/WindowCrossJoinTest.java | 2 +-
.../streaming/examples/join/WindowJoin.java | 36 +++-
.../flink/api/scala/streaming/DataStream.scala | 6 +-
.../api/scala/streaming/FieldsKeySelector.scala | 29 +++
.../scala/streaming/StreamJoinOperator.scala | 188 +++++++++++++++++++
.../scala/streaming/WindowedDataStream.scala | 2 +-
14 files changed, 359 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index f358de9..c826274 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -505,6 +505,15 @@ public class JobGraphBuilder {
}
+ public <IN, OUT> void setInvokable(String id, StreamInvokable<IN, OUT> invokableObject) {
+ invokableObjects.put(id, invokableObject);
+ }
+
+ public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
+ StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
+ typeSerializersOut1.put(id, serializer);
+ }
+
/**
* Sets TypeSerializerWrapper from one vertex to another, used with some
* sinks.
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 65a6c37..39b6460 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -539,7 +539,7 @@ public class ConnectedDataStream<IN1, IN2> {
return invokable;
}
- protected <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
+ public <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
CoWindowFunction<IN1, IN2, OUT> coWindowFunction, TypeInformation<OUT> outTypeInfo,
long windowSize, long slideInterval, TimeStamp<IN1> timestamp1,
TimeStamp<IN2> timestamp2) {
@@ -550,7 +550,7 @@ public class ConnectedDataStream<IN1, IN2> {
if (slideInterval < 1) {
throw new IllegalArgumentException("Slide interval must be positive");
}
-
+
return addCoFunction("coWindowReduce", outTypeInfo, new CoWindowInvokable<IN1, IN2, OUT>(
clean(coWindowFunction), windowSize, slideInterval, timestamp1, timestamp2));
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index a236312..2a0b673 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -92,7 +92,8 @@ public class DataStream<OUT> {
protected List<String> userDefinedNames;
protected boolean selectAll;
protected StreamPartitioner<OUT> partitioner;
- protected final TypeInformation<OUT> typeInfo;
+ @SuppressWarnings("rawtypes")
+ protected TypeInformation typeInfo;
protected List<DataStream<OUT>> mergedStreams;
protected final JobGraphBuilder jobGraphBuilder;
@@ -175,10 +176,18 @@ public class DataStream<OUT> {
*
* @return The type of the datastream.
*/
+ @SuppressWarnings("unchecked")
public TypeInformation<OUT> getType() {
return this.typeInfo;
}
+ @SuppressWarnings("unchecked")
+ public <R> DataStream<R> setType(TypeInformation<R> outType) {
+ jobGraphBuilder.setOutType(id, outType);
+ typeInfo = outType;
+ return (DataStream<R>) this;
+ }
+
public <F> F clean(F f) {
if (getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
ClosureCleaner.clean(f, true);
@@ -979,7 +988,7 @@ public class DataStream<OUT> {
StreamReduceInvokable<OUT> invokable = new StreamReduceInvokable<OUT>(aggregate);
- SingleOutputStreamOperator<OUT, ?> returnStream = transform("reduce", typeInfo, invokable);
+ SingleOutputStreamOperator<OUT, ?> returnStream = transform("reduce", getType(), invokable);
return returnStream;
}
@@ -1077,7 +1086,7 @@ public class DataStream<OUT> {
*/
public DataStreamSink<OUT> addSink(SinkFunction<OUT> sinkFunction) {
- DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", typeInfo);
+ DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", getType());
jobGraphBuilder.addStreamVertex(returnStream.getId(), new SinkInvokable<OUT>(
clean(sinkFunction)), getType(), null, "sink", degreeOfParallelism);
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
index 18b4b75..160ef8d 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
@@ -186,7 +186,7 @@ public class GroupedDataStream<OUT> extends DataStream<OUT> {
GroupedReduceInvokable<OUT> invokable = new GroupedReduceInvokable<OUT>(clean(aggregate),
keySelector);
- SingleOutputStreamOperator<OUT, ?> returnStream = transform("groupReduce", typeInfo,
+ SingleOutputStreamOperator<OUT, ?> returnStream = transform("groupReduce", getType(),
invokable);
return returnStream;
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 016322b..c19517b 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -33,7 +33,7 @@ import org.apache.flink.streaming.state.OperatorState;
/**
* The SingleOutputStreamOperator represents a user defined transformation
* applied on a {@link DataStream} with one predefined output type.
- *
+ *
* @param <OUT>
* Output type of the operator.
* @param <O>
@@ -52,6 +52,13 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
super(dataStream);
}
+ @SuppressWarnings("unchecked")
+ public <R> SingleOutputStreamOperator<R, ?> setType(TypeInformation<R> outType) {
+ jobGraphBuilder.setOutType(id, outType);
+ typeInfo = outType;
+ return (SingleOutputStreamOperator<R, ?>) this;
+ }
+
/**
* Sets the degree of parallelism for this operator. The degree must be 1 or
* more.
@@ -71,7 +78,6 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
return this;
}
-
/**
* Sets the maximum time frequency (ms) for the flushing of the output
* buffer. By default the output buffers flush only when they are full.
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
index 3051587..cbcc1b4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
@@ -23,8 +23,10 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
import org.apache.flink.streaming.util.keys.FieldsKeySelector;
import org.apache.flink.streaming.util.keys.PojoKeySelector;
@@ -122,114 +124,128 @@ public class StreamJoinOperator<I1, I2> extends
}
/**
- * Continues a temporal Join transformation and defines the
- * {@link Tuple} fields of the second join {@link DataStream} that
- * should be used as join keys.<br/>
- * <b>Note: Fields can only be selected as join keys on Tuple
- * DataStreams.</b><br/>
+ * Creates a temporal Join transformation and defines the {@link Tuple}
+ * fields of the second join {@link DataStream} that should be used as
+ * join keys.<br/>
+ * </p> The resulting operator wraps each pair of joining elements in a
+ * Tuple2<I1,I2>(first, second). To use a different wrapping function
+ * use {@link JoinedStream#with(JoinFunction)}
*
* @param fields
* The indexes of the Tuple fields of the second join
* DataStream that should be used as keys.
- * @return An incomplete join. Call {@link FinalizeStreamJoin#with} or
- * {@link FinalizeStreamJoin#withDefault} to complete
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
*/
- public FinalizeStreamJoin<I1, I2> equalTo(int... fields) {
+ public JoinedStream<I1, I2> equalTo(int... fields) {
keys2 = FieldsKeySelector.getSelector(op.input2.getType(), fields);
- return new FinalizeStreamJoin<I1, I2>(this);
+ return createJoinOperator();
}
/**
- * Continues a temporal Join transformation and defines the fields of
- * the second join {@link DataStream} that should be used as join keys.<br/>
+ * Creates a temporal Join transformation and defines the fields of the
+ * second join {@link DataStream} that should be used as join keys. </p>
+ * The resulting operator wraps each pair of joining elements in a
+ * Tuple2<I1,I2>(first, second). To use a different wrapping function
+ * use {@link JoinedStream#with(JoinFunction)}
*
* @param fields
* The fields of the second join DataStream that should be
* used as keys.
- * @return An incomplete join. Call {@link FinalizeStreamJoin#with} or
- * {@link FinalizeStreamJoin#withDefault} to complete
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
*/
- public FinalizeStreamJoin<I1, I2> equalTo(String... fields) {
+ public JoinedStream<I1, I2> equalTo(String... fields) {
this.keys2 = new PojoKeySelector<I2>(op.input2.getType(), fields);
- return new FinalizeStreamJoin<I1, I2>(this);
+ return createJoinOperator();
}
/**
- * Continues a temporal Join transformation and defines a
+ * Creates a temporal Join transformation and defines a
* {@link KeySelector} function for the second join {@link DataStream}
* .</br> The KeySelector function is called for each element of the
* second DataStream and extracts a single key value on which the
- * DataStream is joined. </br>
+ * DataStream is joined. </p> The resulting operator wraps each pair of
+ * joining elements in a Tuple2<I1,I2>(first, second). To use a
+ * different wrapping function use
+ * {@link JoinedStream#with(JoinFunction)}
+ *
*
* @param keySelector
* The KeySelector function which extracts the key values
* from the second DataStream on which it is joined.
- * @return An incomplete join. Call {@link FinalizeStreamJoin#with} or
- * {@link FinalizeStreamJoin#withDefault} to complete
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
*/
- public <K> FinalizeStreamJoin<I1, I2> equalTo(KeySelector<I2, K> keySelector) {
+ public <K> JoinedStream<I1, I2> equalTo(KeySelector<I2, K> keySelector) {
this.keys2 = keySelector;
- return new FinalizeStreamJoin<I1, I2>(this);
+ return createJoinOperator();
}
+ private JoinedStream<I1, I2> createJoinOperator() {
+
+ JoinFunction<I1, I2, Tuple2<I1, I2>> joinFunction = new DefaultJoinFunction<I1, I2>();
+
+ JoinWindowFunction<I1, I2, Tuple2<I1, I2>> joinWindowFunction = getJoinWindowFunction(
+ joinFunction, this);
+
+ TypeInformation<Tuple2<I1, I2>> outType = new TupleTypeInfo<Tuple2<I1, I2>>(
+ op.input1.getType(), op.input2.getType());
+
+ return new JoinedStream<I1, I2>(this, op.input1
+ .groupBy(keys1)
+ .connect(op.input2.groupBy(keys2))
+ .addGeneralWindowCombine(joinWindowFunction, outType, op.windowSize,
+ op.slideInterval, op.timeStamp1, op.timeStamp2));
+ }
}
- public static class FinalizeStreamJoin<I1, I2> {
+ public static class JoinedStream<I1, I2> extends
+ SingleOutputStreamOperator<Tuple2<I1, I2>, JoinedStream<I1, I2>> {
private final JoinPredicate<I1, I2> predicate;
- private FinalizeStreamJoin(JoinPredicate<I1, I2> predicate) {
+ private JoinedStream(JoinPredicate<I1, I2> predicate, DataStream<Tuple2<I1, I2>> ds) {
+ super(ds);
this.predicate = predicate;
}
/**
* Completes a stream join. </p> The resulting operator wraps each pair
- * of joining elements into a {@link Tuple2}, with the element of the
- * first input being the first field of the tuple and the element of the
- * second input being the second field of the tuple.
- *
- * @return The joined data stream.
- */
- public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> withDefault() {
- return createJoinOperator(new DefaultJoinFunction<I1, I2>());
- }
-
- /**
- * Completes a stream join. </p> The resulting operator wraps each pair
* of joining elements using the user defined {@link JoinFunction}
*
* @return The joined data stream.
*/
public <OUT> SingleOutputStreamOperator<OUT, ?> with(JoinFunction<I1, I2, OUT> joinFunction) {
- return createJoinOperator(joinFunction);
- }
- private <OUT> SingleOutputStreamOperator<OUT, ?> createJoinOperator(
- JoinFunction<I1, I2, OUT> joinFunction) {
-
- JoinWindowFunction<I1, I2, OUT> joinWindowFunction = new JoinWindowFunction<I1, I2, OUT>(
- predicate.keys1, predicate.keys2, joinFunction);
+ TypeInformation<OUT> outType = TypeExtractor.getJoinReturnTypes(joinFunction,
+ predicate.op.input1.getType(), predicate.op.input2.getType());
- StreamJoinOperator<I1, I2> op = predicate.op;
+ CoWindowInvokable<I1, I2, OUT> invokable = new CoWindowInvokable<I1, I2, OUT>(
+ getJoinWindowFunction(joinFunction, predicate), predicate.op.windowSize,
+ predicate.op.slideInterval, predicate.op.timeStamp1, predicate.op.timeStamp2);
- TypeInformation<OUT> outType = TypeExtractor.getJoinReturnTypes(joinFunction,
- op.input1.getType(), op.input2.getType());
+ jobGraphBuilder.setInvokable(id, invokable);
- return op.input1.connect(op.input2).addGeneralWindowCombine(joinWindowFunction,
- outType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2);
+ return setType(outType);
}
}
- public static final class DefaultJoinFunction<T1, T2> implements
- JoinFunction<T1, T2, Tuple2<T1, T2>> {
+ public static final class DefaultJoinFunction<I1, I2> implements
+ JoinFunction<I1, I2, Tuple2<I1, I2>> {
private static final long serialVersionUID = 1L;
- private final Tuple2<T1, T2> outTuple = new Tuple2<T1, T2>();
+ private final Tuple2<I1, I2> outTuple = new Tuple2<I1, I2>();
@Override
- public Tuple2<T1, T2> join(T1 first, T2 second) throws Exception {
+ public Tuple2<I1, I2> join(I1 first, I2 second) throws Exception {
outTuple.f0 = first;
outTuple.f1 = second;
return outTuple;
}
}
+
+ public static <I1, I2, OUT> JoinWindowFunction<I1, I2, OUT> getJoinWindowFunction(
+ JoinFunction<I1, I2, OUT> joinFunction, JoinPredicate<I1, I2> predicate) {
+ return new JoinWindowFunction<I1, I2, OUT>(predicate.keys1, predicate.keys2, joinFunction);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
index e71b18c..c8ad533 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamProjection.java
@@ -56,7 +56,7 @@ public class StreamProjection<IN> {
protected StreamProjection(DataStream<IN> dataStream, int[] fieldIndexes) {
this.dataStream = dataStream;
this.fieldIndexes = fieldIndexes;
- this.inTypeInfo = dataStream.typeInfo;
+ this.inTypeInfo = dataStream.getType();
if (!inTypeInfo.isTupleType()) {
throw new RuntimeException("Only Tuple DataStreams can be projected");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java
index d785109..171ddc9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/keys/FieldsKeySelector.java
@@ -56,12 +56,13 @@ public abstract class FieldsKeySelector<IN> implements KeySelector<IN, Object> {
protected Object key;
protected boolean simpleKey;
- public static Class<?>[] tupleClasses = new Class[] { Tuple1.class, Tuple2.class, Tuple3.class,
- Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class, Tuple9.class,
- Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class, Tuple14.class,
- Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class, Tuple19.class,
- Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class, Tuple24.class,
- Tuple25.class };
+ @SuppressWarnings("unchecked")
+ public static Class<? extends Tuple>[] tupleClasses = new Class[] { Tuple1.class, Tuple2.class,
+ Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
+ Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
+ Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
+ Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
+ Tuple24.class, Tuple25.class };
public FieldsKeySelector(int... fields) {
this.keyFields = fields;
@@ -73,7 +74,7 @@ public abstract class FieldsKeySelector<IN> implements KeySelector<IN, Object> {
}
try {
- key = (Tuple) tupleClasses[fields.length - 1].newInstance();
+ key = tupleClasses[fields.length - 1].newInstance();
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
index ffdfbc3..d8cdfa5 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -96,7 +96,7 @@ public class WindowCrossJoinTest implements Serializable {
DataStream<Integer> inStream2 = env.fromCollection(in2);
inStream1.join(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
- .where(0).equalTo(0).withDefault().addSink(new JoinResultSink());
+ .where(0).equalTo(0).addSink(new JoinResultSink());
inStream1.cross(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
.with(new CrossFunction<Tuple2<Integer,String>, Integer, Tuple2<Tuple2<Integer,String>, Integer>>() {
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
index 2586e3c..897ad48 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
@@ -19,7 +19,9 @@ package org.apache.flink.streaming.examples.join;
import java.util.Random;
+import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -34,7 +36,7 @@ import org.apache.flink.util.Collector;
* his example will join two streams with a sliding window. One which emits
* grades and one which emits salaries of people.
* </p>
- *
+ *
* <p>
* This example shows how to:
* <ul>
@@ -63,13 +65,13 @@ public class WindowJoin {
// apply a temporal join over the two stream based on the names over one
// second windows
- DataStream<Tuple2<Tuple2<String, Integer>, Tuple2<String, Integer>>> joinedStream = grades
- .join(salaries)
- .onWindow(1000)
- .where(0)
- .equalTo(0)
- .withDefault();
-
+ DataStream<Tuple3<String, Integer, Integer>> joinedStream = grades
+ .join(salaries)
+ .onWindow(1000)
+ .where(0)
+ .equalTo(0)
+ .with(new MyJoinFunction());
+
// emit result
if (fileOutput) {
joinedStream.writeAsText(outputPath, 1);
@@ -141,6 +143,24 @@ public class WindowJoin {
}
}
+ public static class MyJoinFunction
+ implements
+ JoinFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Integer, Integer>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private Tuple3<String, Integer, Integer> joined = new Tuple3<String, Integer, Integer>();
+
+ @Override
+ public Tuple3<String, Integer, Integer> join(Tuple2<String, Integer> first,
+ Tuple2<String, Integer> second) throws Exception {
+ joined.f0 = first.f0;
+ joined.f1 = first.f1;
+ joined.f2 = second.f1;
+ return joined;
+ }
+ }
+
// *************************************************************************
// UTIL METHODS
// *************************************************************************
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index a117412..871fede 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -108,7 +108,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def groupBy(fields: Int*): DataStream[T] =
- new DataStream[T](javaStream.groupBy(fields: _*))
+ new DataStream[T](javaStream.groupBy(new FieldsKeySelector[T](fields: _*)))
/**
* Groups the elements of a DataStream by the given field expressions to
@@ -138,7 +138,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def partitionBy(fields: Int*): DataStream[T] =
- new DataStream[T](javaStream.partitionBy(fields: _*))
+ new DataStream[T](javaStream.partitionBy(new FieldsKeySelector[T](fields: _*)))
/**
* Sets the partitioning of the DataStream so that the output is
@@ -458,6 +458,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
split(selector)
}
+ def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] = new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
+
/**
* Writes a DataStream to the standard output stream (stdout). For each
* element of the DataStream the result of .toString is
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
new file mode 100644
index 0000000..4223512
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
@@ -0,0 +1,29 @@
+package org.apache.flink.api.scala.streaming
+
+import org.apache.flink.streaming.util.keys.{ FieldsKeySelector => JavaSelector }
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.tuple.Tuple
+
+class FieldsKeySelector[IN](fields: Int*) extends KeySelector[IN, Tuple] {
+
+ val t: Tuple = JavaSelector.tupleClasses(fields.length - 1).newInstance()
+
+ override def getKey(value: IN): Tuple =
+
+ value match {
+ case prod: Product => {
+ for (i <- 0 to fields.length - 1) {
+ t.setField(prod.productElement(fields(i)), i)
+ }
+ t
+ }
+ case tuple: Tuple => {
+ for (i <- 0 to fields.length - 1) {
+ t.setField(tuple.getField(fields(i)), i)
+ }
+ t
+ }
+ case _ => throw new RuntimeException("Only tuple types are supported")
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
new file mode 100644
index 0000000..93950a2
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+
+import org.apache.flink.api.common.functions.JoinFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.scala.ClosureCleaner
+import org.apache.flink.api.scala.typeutils.CaseClassSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.TemporalOperator
+import org.apache.flink.streaming.api.function.co.JoinWindowFunction
+import org.apache.flink.streaming.util.keys.PojoKeySelector
+import scala.reflect.ClassTag
+import org.apache.commons.lang.Validate
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
+
+class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
+
+ override def createNextWindowOperator() = {
+ new StreamJoinOperator.JoinWindow[I1, I2](this)
+ }
+}
+
+object StreamJoinOperator {
+
+ private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
+ ClosureCleaner.clean(f, checkSerializable)
+ f
+ }
+
+ class JoinWindow[I1, I2](op: StreamJoinOperator[I1, I2]) {
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the fields in the first stream to be used as keys for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where(fields: Int*) = {
+ new JoinPredicate[I1, I2](op, new FieldsKeySelector[I1](fields: _*))
+ }
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the fields in the first stream to be used as keys for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where(firstField: String, otherFields: String*) = {
+ new JoinPredicate[I1, I2](op, new PojoKeySelector[I1](op.input1.getType(), (firstField +: otherFields): _*))
+ }
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the keyselector function that will be used to extract keys from the first stream
+ * for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where[K: TypeInformation](fun: (I1) => K) = {
+ val keyType = implicitly[TypeInformation[K]]
+ val keyExtractor = new KeySelector[I1, K] {
+ val cleanFun = op.input1.clean(fun)
+ def getKey(in: I1) = cleanFun(in)
+ }
+ new JoinPredicate[I1, I2](op, keyExtractor)
+ }
+
+ }
+
+ class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2], private[flink] val keys1: KeySelector[I1, _]) {
+ private[flink] var keys2: KeySelector[I2, _] = null
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.with(...)
+ */
+ def equalTo(fields: Int*): JoinedStream[I1, I2] = {
+ finish(new FieldsKeySelector[I2](fields: _*))
+ }
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.with(...)
+ */
+ def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] = {
+ finish(new PojoKeySelector[I2](op.input2.getType(), (firstField +: otherFields): _*))
+ }
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.with(...)
+ */
+ def equalTo[K: TypeInformation](fun: (I2) => K): JoinedStream[I1, I2] = {
+ val keyType = implicitly[TypeInformation[K]]
+ val keyExtractor = new KeySelector[I2, K] {
+ val cleanFun = op.input1.clean(fun)
+ def getKey(in: I2) = cleanFun(in)
+ }
+ finish(keyExtractor)
+ }
+
+ private def finish(keys2: KeySelector[I2, _]): JoinedStream[I1, I2] = {
+ this.keys2 = keys2
+ new JoinedStream[I1, I2](this, createJoinOperator())
+ }
+
+ private def createJoinOperator(): JavaStream[(I1, I2)] = {
+
+ val returnType = new CaseClassTypeInfo[(I1, I2)](
+
+ classOf[(I1, I2)], Seq(op.input1.getType, op.input2.getType), Array("_1", "_2")) {
+
+ override def createSerializer: TypeSerializer[(I1, I2)] = {
+ val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
+ for (i <- 0 until getArity) {
+ fieldSerializers(i) = types(i).createSerializer
+ }
+
+ new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
+ override def createInstance(fields: Array[AnyRef]) = {
+ (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
+ }
+ }
+ }
+ }
+
+ return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2)).addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
+ returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+ }
+ }
+
+ class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends DataStream[(I1, I2)](javaStream) {
+
+ private val op = jp.op
+
+ def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
+
+ val invokable = new CoWindowInvokable[I1, I2, R](
+ clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+
+ javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(), invokable)
+
+ new DataStream[R](javaStream.setType(implicitly[TypeInformation[R]]))
+ }
+ }
+
+ private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2], joinFunction: (I1, I2) => R) = {
+ Validate.notNull(joinFunction, "Join function must not be null.")
+
+ val joinFun = new JoinFunction[I1, I2, R] {
+
+ val cleanFun = clean(joinFunction)
+
+ override def join(first: I1, second: I2): R = {
+ cleanFun(first, second)
+ }
+ }
+
+ new JoinWindowFunction[I1, I2, R](jp.keys1, jp.keys2, joinFun)
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/739fdf0b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
index ff89a47..c686497 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -61,7 +61,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
*
*/
def groupBy(fields: Int*): WindowedDataStream[T] =
- new WindowedDataStream[T](javaStream.groupBy(fields: _*))
+ new WindowedDataStream[T](javaStream.groupBy(new FieldsKeySelector[T](fields: _*)))
/**
* Groups the elements of the WindowedDataStream using the given
[12/36] flink git commit: [scala] [streaming] Fixed scala formatting
Posted by mb...@apache.org.
[scala] [streaming] Fixed scala formatting
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1f7b6eaa
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1f7b6eaa
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1f7b6eaa
Branch: refs/heads/release-0.8
Commit: 1f7b6eaaa979e91799a047f29881f82666c12b19
Parents: f7291ea
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Dec 21 01:05:40 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:58:54 2015 +0100
----------------------------------------------------------------------
.../flink/api/scala/streaming/DataStream.scala | 93 +++++++++++++-------
.../api/scala/streaming/FieldsKeySelector.scala | 2 +-
.../api/scala/streaming/SplitDataStream.scala | 17 ++--
.../scala/streaming/StreamCrossOperator.scala | 25 ++++--
.../streaming/StreamExecutionEnvironment.scala | 16 ++--
.../scala/streaming/StreamJoinOperator.scala | 29 +++---
.../scala/streaming/WindowedDataStream.scala | 31 ++++---
7 files changed, 136 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1f7b6eaa/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index ecf5615..0cf4a60 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -82,7 +82,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
javaStream match {
case ds: SingleOutputStreamOperator[_, _] => ds.setParallelism(dop)
case _ =>
- throw new UnsupportedOperationException("Operator " + javaStream.toString + " cannot have " +
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " cannot " +
+ "have " +
"parallelism.")
}
this
@@ -94,7 +95,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
def getParallelism: Int = javaStream match {
case op: SingleOutputStreamOperator[_, _] => op.getParallelism
case _ =>
- throw new UnsupportedOperationException("Operator " + javaStream.toString + " does not have " +
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " does not have" +
+ " " +
"parallelism.")
}
@@ -139,7 +141,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
/**
* Sets the partitioning of the DataStream so that the output is
- * partitioned by the selected fields. This setting only effects the how the outputs will be distributed between the parallel instances of the next processing operator.
+ * partitioned by the selected fields. This setting only effects the how the outputs will be
+ * distributed between the parallel instances of the next processing operator.
*
*/
def partitionBy(fields: Int*): DataStream[T] =
@@ -147,7 +150,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
/**
* Sets the partitioning of the DataStream so that the output is
- * partitioned by the selected fields. This setting only effects the how the outputs will be distributed between the parallel instances of the next processing operator.
+ * partitioned by the selected fields. This setting only effects the how the outputs will be
+ * distributed between the parallel instances of the next processing operator.
*
*/
def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
@@ -155,7 +159,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
/**
* Sets the partitioning of the DataStream so that the output is
- * partitioned by the given Key. This setting only effects the how the outputs will be distributed between the parallel instances of the next processing operator.
+ * partitioned by the given Key. This setting only effects the how the outputs will be
+ * distributed between the parallel instances of the next processing operator.
*
*/
def partitionBy[K: TypeInformation](fun: T => K): DataStream[T] = {
@@ -222,7 +227,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*
*/
- def iterate(stepFunction: DataStream[T] => (DataStream[T], DataStream[T]), maxWaitTimeMillis: Long = 0): DataStream[T] = {
+ def iterate(stepFunction: DataStream[T] => (DataStream[T], DataStream[T]), maxWaitTimeMillis:
+ Long = 0): DataStream[T] = {
val iterativeStream = javaStream.iterate(maxWaitTimeMillis)
val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
@@ -252,19 +258,24 @@ class DataStream[T](javaStream: JavaStream[T]) {
/**
* Applies an aggregation that that gives the current minimum element of the data stream by
- * the given position. When equality, the user can set to get the first or last element with the minimal value.
+ * the given position. When equality, the user can set to get the first or last element with
+ * the minimal value.
*
*/
- def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MINBY, position, first)
+ def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType
+ .MINBY, position, first)
/**
* Applies an aggregation that that gives the current maximum element of the data stream by
- * the given position. When equality, the user can set to get the first or last element with the maximal value.
+ * the given position. When equality, the user can set to get the first or last element with
+ * the maximal value.
*
*/
- def maxBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MAXBY, position, first)
+ def maxBy(position: Int, first: Boolean = true): DataStream[T] =
+ aggregate(AggregationType.MAXBY, position, first)
- private def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true): DataStream[T] = {
+ private def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true):
+ DataStream[T] = {
val jStream = javaStream.asInstanceOf[JavaStream[Product]]
val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
@@ -272,15 +283,18 @@ class DataStream[T](javaStream: JavaStream[T]) {
val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
val reducer = aggregationType match {
- case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).getTypeClass()));
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).
+ getTypeClass()));
case _ => new agg.ProductComparableAggregator(aggregationType, first)
}
val invokable = jStream match {
- case groupedStream: GroupedDataStream[_] => new GroupedReduceInvokable(reducer, groupedStream.getKeySelector())
+ case groupedStream: GroupedDataStream[_] => new GroupedReduceInvokable(reducer,
+ groupedStream.getKeySelector())
case _ => new StreamReduceInvokable(reducer)
}
- new DataStream[Product](jStream.transform("aggregation", jStream.getType(), invokable)).asInstanceOf[DataStream[T]]
+ new DataStream[Product](jStream.transform("aggregation", jStream.getType(),
+ invokable)).asInstanceOf[DataStream[T]]
}
/**
@@ -288,7 +302,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* received records.
*
*/
- def count: DataStream[Long] = new DataStream[java.lang.Long](javaStream.count()).asInstanceOf[DataStream[Long]]
+ def count: DataStream[Long] = new DataStream[java.lang.Long](
+ javaStream.count()).asInstanceOf[DataStream[Long]]
/**
* Creates a new DataStream by applying the given function to every element of this DataStream.
@@ -302,7 +317,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
def map(in: T): R = cleanFun(in)
}
- new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper)))
+ new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]],
+ new MapInvokable[T, R](mapper)))
}
/**
@@ -313,7 +329,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
throw new NullPointerException("Map function must not be null.")
}
- new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper)))
+ new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]],
+ new MapInvokable[T, R](mapper)))
}
/**
@@ -324,7 +341,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
if (flatMapper == null) {
throw new NullPointerException("FlatMap function must not be null.")
}
- new DataStream[R](javaStream.transform("flatMap", implicitly[TypeInformation[R]], new FlatMapInvokable[T, R](flatMapper)))
+ new DataStream[R](javaStream.transform("flatMap", implicitly[TypeInformation[R]],
+ new FlatMapInvokable[T, R](flatMapper)))
}
/**
@@ -358,22 +376,24 @@ class DataStream[T](javaStream: JavaStream[T]) {
}
/**
- * Creates a new [[DataStream]] by reducing the elements of this DataStream using an associative reduce
- * function.
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream
+ * using an associative reduce function.
*/
def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
if (reducer == null) {
throw new NullPointerException("Reduce function must not be null.")
}
javaStream match {
- case ds: GroupedDataStream[_] => new DataStream[T](javaStream.transform("reduce", javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector())))
- case _ => new DataStream[T](javaStream.transform("reduce", javaStream.getType(), new StreamReduceInvokable[T](reducer)))
+ case ds: GroupedDataStream[_] => new DataStream[T](javaStream.transform("reduce",
+ javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector())))
+ case _ => new DataStream[T](javaStream.transform("reduce", javaStream.getType(),
+ new StreamReduceInvokable[T](reducer)))
}
}
/**
- * Creates a new [[DataStream]] by reducing the elements of this DataStream using an associative reduce
- * function.
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream
+ * using an associative reduce function.
*/
def reduce(fun: (T, T) => T): DataStream[T] = {
if (fun == null) {
@@ -421,7 +441,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* the trigger and eviction policies please use to
* window(List(triggers), List(evicters))
*/
- def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] = new WindowedDataStream[T](javaStream.window(windowingHelper: _*))
+ def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
+ new WindowedDataStream[T](javaStream.window(windowingHelper: _*))
/**
* Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
@@ -430,7 +451,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* use-cases please refer to window(WindowingHelper[_]*)
*
*/
- def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]): WindowedDataStream[T] = new WindowedDataStream[T](javaStream.window(triggers, evicters))
+ def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]):
+ WindowedDataStream[T] = new WindowedDataStream[T](javaStream.window(triggers, evicters))
/**
*
@@ -473,7 +495,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* to use custom join function.
*
*/
- def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] = new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
+ def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] =
+ new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
/**
* Initiates a temporal cross transformation that builds all pair
@@ -487,7 +510,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* to use custom join function.
*
*/
- def cross[R](stream: DataStream[R]): StreamCrossOperator[T, R] = new StreamCrossOperator[T, R](javaStream, stream.getJavaStream)
+ def cross[R](stream: DataStream[R]): StreamCrossOperator[T, R] =
+ new StreamCrossOperator[T, R](javaStream, stream.getJavaStream)
/**
* Writes a DataStream to the standard output stream (stdout). For each
@@ -504,7 +528,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* is written.
*
*/
- def writeAsText(path: String, millis: Long = 0): DataStream[T] = new DataStream[T](javaStream.writeAsText(path, millis))
+ def writeAsText(path: String, millis: Long = 0): DataStream[T] =
+ new DataStream[T](javaStream.writeAsText(path, millis))
/**
* Writes a DataStream to the file specified by path in text format. The
@@ -513,7 +538,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* is written.
*
*/
- def writeAsCsv(path: String, millis: Long = 0): DataStream[T] = new DataStream[T](javaStream.writeAsCsv(path, millis))
+ def writeAsCsv(path: String, millis: Long = 0): DataStream[T] =
+ new DataStream[T](javaStream.writeAsCsv(path, millis))
/**
* Adds the given sink to this DataStream. Only streams with sinks added
@@ -521,7 +547,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
* method is called.
*
*/
- def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] = new DataStream[T](javaStream.addSink(sinkFuntion))
+ def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] =
+ new DataStream[T](javaStream.addSink(sinkFuntion))
/**
* Adds the given sink to this DataStream. Only streams with sinks added
@@ -540,4 +567,4 @@ class DataStream[T](javaStream: JavaStream[T]) {
this.addSink(sinkFunction)
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f7b6eaa/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
index d7c9f96..b50d346 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
@@ -44,4 +44,4 @@ class FieldsKeySelector[IN](fields: Int*) extends KeySelector[IN, Tuple] {
case _ => throw new RuntimeException("Only tuple types are supported")
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f7b6eaa/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
index 0b0cce5..82a5c70 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
@@ -1,12 +1,13 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -46,4 +47,4 @@ class SplitDataStream[T](javaStream: SplitJavaStream[T]) {
*/
def selectAll(): DataStream[T] = new DataStream[T](javaStream.selectAll())
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f7b6eaa/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
index 5dfbc3b..5f579e5 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
@@ -35,11 +35,13 @@ import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.api.function.co.CrossWindowFunction
import org.apache.flink.api.common.functions.CrossFunction
-class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
+class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
+ TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
override def createNextWindowOperator(): StreamCrossOperator.CrossWindow[I1, I2] = {
- val crossWindowFunction = StreamCrossOperator.getCrossWindowFunction(this, (l: I1, r: I2) => (l, r))
+ val crossWindowFunction = StreamCrossOperator.getCrossWindowFunction(this,
+ (l: I1, r: I2) => (l, r))
val returnType = new CaseClassTypeInfo[(I1, I2)](
@@ -69,24 +71,31 @@ class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extend
}
object StreamCrossOperator {
- private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2], javaStream: JavaStream[(I1, I2)]) extends DataStream[(I1, I2)](javaStream) {
+ private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2],
+ javaStream: JavaStream[(I1, I2)]) extends
+ DataStream[(I1, I2)](javaStream) {
/**
- * Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf call will be emitted.
+ * Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf
+ * call will be emitted.
*
*/
def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
val invokable = new CoWindowInvokable[I1, I2, R](
- clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+ clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
+ op.timeStamp2)
- javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(), invokable)
+ javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+ invokable)
new DataStream[R](javaStream.setType(implicitly[TypeInformation[R]]))
}
}
- private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2], crossFunction: (I1, I2) => R): CrossWindowFunction[I1, I2, R] = {
+ private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2],
+ crossFunction: (I1, I2) => R):
+ CrossWindowFunction[I1, I2, R] = {
Validate.notNull(crossFunction, "Join function must not be null.")
val crossFun = new CrossFunction[I1, I2, R] {
@@ -100,4 +109,4 @@ object StreamCrossOperator {
new CrossWindowFunction[I1, I2, R](crossFun)
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f7b6eaa/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index 340ecc1..55f7c6c 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -117,7 +117,8 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*
*/
def generateSequence(from: Long, to: Long): DataStream[Long] = {
- new DataStream[java.lang.Long](javaEnv.generateSequence(from, to)).asInstanceOf[DataStream[Long]]
+ new DataStream[java.lang.Long](javaEnv.generateSequence(from, to)).
+ asInstanceOf[DataStream[Long]]
}
/**
@@ -147,7 +148,8 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
"elements", typeInfo);
javaEnv.getJobGraphBuilder.addStreamVertex(returnStream.getId(),
- new SourceInvokable[T](new FromElementsFunction[T](scala.collection.JavaConversions.asJavaCollection(data))), null, typeInfo,
+ new SourceInvokable[T](new FromElementsFunction[T](scala.collection.JavaConversions
+ .asJavaCollection(data))), null, typeInfo,
"source", 1);
new DataStream(returnStream)
}
@@ -204,7 +206,8 @@ object StreamExecutionEnvironment {
* of parallelism of the local environment is the number of hardware contexts (CPU cores/threads).
*/
def createLocalEnvironment(
- degreeOfParallelism: Int = Runtime.getRuntime.availableProcessors()): StreamExecutionEnvironment = {
+ degreeOfParallelism: Int = Runtime.getRuntime.availableProcessors()):
+ StreamExecutionEnvironment = {
new StreamExecutionEnvironment(JavaEnv.createLocalEnvironment(degreeOfParallelism))
}
@@ -223,7 +226,8 @@ object StreamExecutionEnvironment {
* those must be
* provided in the JAR files.
*/
- def createRemoteEnvironment(host: String, port: Int, jarFiles: String*): StreamExecutionEnvironment = {
+ def createRemoteEnvironment(host: String, port: Int, jarFiles: String*):
+ StreamExecutionEnvironment = {
new StreamExecutionEnvironment(JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*))
}
@@ -251,4 +255,4 @@ object StreamExecutionEnvironment {
javaEnv.setDegreeOfParallelism(degreeOfParallelism)
new StreamExecutionEnvironment(javaEnv)
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f7b6eaa/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
index fff5e86..7a39da5 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
@@ -33,7 +33,8 @@ import scala.reflect.ClassTag
import org.apache.commons.lang.Validate
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
-class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
+class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
+TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
override def createNextWindowOperator() = {
new StreamJoinOperator.JoinWindow[I1, I2](this)
@@ -61,7 +62,8 @@ object StreamJoinOperator {
* to define the second key.
*/
def where(firstField: String, otherFields: String*) = {
- new JoinPredicate[I1, I2](op, new PojoKeySelector[I1](op.input1.getType(), (firstField +: otherFields): _*))
+ new JoinPredicate[I1, I2](op, new PojoKeySelector[I1](op.input1.getType(),
+ (firstField +: otherFields): _*))
}
/**
@@ -82,7 +84,8 @@ object StreamJoinOperator {
}
- class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2], private[flink] val keys1: KeySelector[I1, _]) {
+ class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2],
+ private[flink] val keys1: KeySelector[I1, _]) {
private[flink] var keys2: KeySelector[I2, _] = null
/**
@@ -145,30 +148,36 @@ object StreamJoinOperator {
}
}
- return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2)).addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
+ return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2))
+ .addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
}
}
- class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends DataStream[(I1, I2)](javaStream) {
+ class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends
+ DataStream[(I1, I2)](javaStream) {
private val op = jp.op
/**
- * Sets a wrapper for the joined elements. For each joined pair, the result of the udf call will be emitted.
+ * Sets a wrapper for the joined elements. For each joined pair, the result of the
+ * udf call will be emitted.
*/
def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
val invokable = new CoWindowInvokable[I1, I2, R](
- clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+ clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
+ op.timeStamp2)
- javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(), invokable)
+ javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+ invokable)
new DataStream[R](javaStream.setType(implicitly[TypeInformation[R]]))
}
}
- private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2], joinFunction: (I1, I2) => R) = {
+ private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2],
+ joinFunction: (I1, I2) => R) = {
Validate.notNull(joinFunction, "Join function must not be null.")
val joinFun = new JoinFunction[I1, I2, R] {
@@ -183,4 +192,4 @@ object StreamJoinOperator {
new JoinWindowFunction[I1, I2, R](jp.keys1, jp.keys2, joinFun)
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/1f7b6eaa/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
index c037305..8c763fc 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -52,7 +52,8 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
* This controls how often the user defined function will be triggered on
* the window.
*/
- def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] = new WindowedDataStream[T](javaStream.every(windowingHelper: _*))
+ def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
+ new WindowedDataStream[T](javaStream.every(windowingHelper: _*))
/**
* Groups the elements of the WindowedDataStream using the given
@@ -126,12 +127,14 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
/**
* Applies a reduceGroup transformation on the windowed data stream by reducing
- * the current window at every trigger. In contrast with the simple binary reduce operator, groupReduce exposes the whole window through the Iterable interface.
+ * the current window at every trigger. In contrast with the simple binary reduce operator,
+ * groupReduce exposes the whole window through the Iterable interface.
* </br>
* </br>
* Whenever possible try to use reduce instead of groupReduce for increased efficiency
*/
- def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]): DataStream[R] = {
+ def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
+ DataStream[R] = {
if (reducer == null) {
throw new NullPointerException("GroupReduce function must not be null.")
}
@@ -140,12 +143,14 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
/**
* Applies a reduceGroup transformation on the windowed data stream by reducing
- * the current window at every trigger. In contrast with the simple binary reduce operator, groupReduce exposes the whole window through the Iterable interface.
+ * the current window at every trigger. In contrast with the simple binary reduce operator,
+ * groupReduce exposes the whole window through the Iterable interface.
* </br>
* </br>
* Whenever possible try to use reduce instead of groupReduce for increased efficiency
*/
- def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit): DataStream[R] = {
+ def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
+ DataStream[R] = {
if (fun == null) {
throw new NullPointerException("GroupReduce function must not be null.")
}
@@ -181,16 +186,19 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
* the given position. When equality, returns the first.
*
*/
- def maxBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MAXBY, position, first)
+ def maxBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MAXBY,
+ position, first)
/**
* Applies an aggregation that that gives the minimum element of the window by
* the given position. When equality, returns the first.
*
*/
- def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MINBY, position, first)
+ def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MINBY,
+ position, first)
- def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true): DataStream[T] = {
+ def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true):
+ DataStream[T] = {
val jStream = javaStream.asInstanceOf[JavaWStream[Product]]
val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
@@ -198,11 +206,12 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
val reducer = aggregationType match {
- case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).getTypeClass()));
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(
+ outType.getTypeAt(position).getTypeClass()));
case _ => new agg.ProductComparableAggregator(aggregationType, first)
}
new DataStream[Product](jStream.reduce(reducer)).asInstanceOf[DataStream[T]]
}
-}
\ No newline at end of file
+}
[13/36] flink git commit: [scala] [streaming] Added groupBy support
for case class fields
Posted by mb...@apache.org.
[scala] [streaming] Added groupBy support for case class fields
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/14134842
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/14134842
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/14134842
Branch: refs/heads/release-0.8
Commit: 141348426eeae2ea0e4404a0d8cff8182c601886
Parents: 1f7b6ea
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Dec 21 03:54:30 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:59:43 2015 +0100
----------------------------------------------------------------------
.../aggregation/ComparableAggregator.java | 8 +---
.../scala/streaming/CaseClassKeySelector.scala | 45 ++++++++++++++++++++
.../flink/api/scala/streaming/DataStream.scala | 19 +++++++--
.../api/scala/streaming/FieldsKeySelector.scala | 23 +++-------
.../scala/streaming/StreamJoinOperator.scala | 19 ++++++---
.../scala/streaming/WindowedDataStream.scala | 11 ++++-
.../api/scala/typeutils/CaseClassTypeInfo.scala | 1 +
7 files changed, 92 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/14134842/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
index 5fb8f62..7ea7ba1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
@@ -143,8 +143,6 @@ public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
} else {
if (c == 1) {
Array.set(array2, position, v1);
- } else {
- Array.set(array2, position, v2);
}
return array2;
@@ -230,10 +228,8 @@ public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
} else {
if (c == 1) {
keyFields[0].set(value2, field1);
- } else {
- keyFields[0].set(value2, field2);
- }
-
+ }
+
return value2;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/14134842/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala
new file mode 100644
index 0000000..63410a9
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/CaseClassKeySelector.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala.streaming
+
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import java.util.ArrayList
+import org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor
+import org.apache.flink.api.java.functions.KeySelector
+
+class CaseClassKeySelector[T <: Product](@transient val typeInfo: CaseClassTypeInfo[T],
+ val keyFields: String*) extends KeySelector[T, Seq[Any]] {
+
+ val numOfKeys: Int = keyFields.length;
+
+ @transient val fieldDescriptors = new ArrayList[FlatFieldDescriptor]();
+ for (field <- keyFields) {
+ typeInfo.getKey(field, 0, fieldDescriptors);
+ }
+
+ val logicalKeyPositions = new Array[Int](numOfKeys)
+ val orders = new Array[Boolean](numOfKeys)
+
+ for (i <- 0 to numOfKeys - 1) {
+ logicalKeyPositions(i) = fieldDescriptors.get(i).getPosition();
+ }
+
+ def getKey(value: T): Seq[Any] = {
+ for (i <- 0 to numOfKeys - 1) yield value.productElement(logicalKeyPositions(i))
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/14134842/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 0cf4a60..6df4b25 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -49,6 +49,7 @@ import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
import com.amazonaws.services.cloudfront_2012_03_15.model.InvalidArgumentException
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
class DataStream[T](javaStream: JavaStream[T]) {
@@ -122,9 +123,14 @@ class DataStream[T](javaStream: JavaStream[T]) {
* be used with grouped operators like grouped reduce or grouped aggregations
*
*/
- def groupBy(firstField: String, otherFields: String*): DataStream[T] =
- new DataStream[T](javaStream.groupBy(firstField +: otherFields.toArray: _*))
-
+ def groupBy(firstField: String, otherFields: String*): DataStream[T] =
+ javaStream.getType() match {
+ case ccInfo: CaseClassTypeInfo[T] => new DataStream[T](javaStream.groupBy(
+ new CaseClassKeySelector[T](ccInfo, firstField +: otherFields.toArray: _*)))
+ case _ => new DataStream[T](javaStream.groupBy(
+ firstField +: otherFields.toArray: _*))
+ }
+
/**
* Groups the elements of a DataStream by the given K key to
* be used with grouped operators like grouped reduce or grouped aggregations
@@ -155,7 +161,12 @@ class DataStream[T](javaStream: JavaStream[T]) {
*
*/
def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
- new DataStream[T](javaStream.partitionBy(firstField +: otherFields.toArray: _*))
+ javaStream.getType() match {
+ case ccInfo: CaseClassTypeInfo[T] => new DataStream[T](javaStream.partitionBy(
+ new CaseClassKeySelector[T](ccInfo, firstField +: otherFields.toArray: _*)))
+ case _ => new DataStream[T](javaStream.partitionBy(
+ firstField +: otherFields.toArray: _*))
+ }
/**
* Sets the partitioning of the DataStream so that the output is
http://git-wip-us.apache.org/repos/asf/flink/blob/14134842/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
index b50d346..bc79fca 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/FieldsKeySelector.scala
@@ -22,25 +22,16 @@ import org.apache.flink.streaming.util.keys.{ FieldsKeySelector => JavaSelector
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.java.tuple.Tuple
-class FieldsKeySelector[IN](fields: Int*) extends KeySelector[IN, Tuple] {
+class FieldsKeySelector[IN](fields: Int*) extends KeySelector[IN, Seq[Any]] {
- val t: Tuple = JavaSelector.tupleClasses(fields.length - 1).newInstance()
-
- override def getKey(value: IN): Tuple =
+ override def getKey(value: IN): Seq[Any] =
value match {
- case prod: Product => {
- for (i <- 0 to fields.length - 1) {
- t.setField(prod.productElement(fields(i)), i)
- }
- t
- }
- case tuple: Tuple => {
- for (i <- 0 to fields.length - 1) {
- t.setField(tuple.getField(fields(i)), i)
- }
- t
- }
+ case prod: Product =>
+ for (i <- 0 to fields.length - 1) yield prod.productElement(fields(i))
+ case tuple: Tuple =>
+ for (i <- 0 to fields.length - 1) yield tuple.getField(fields(i))
+
case _ => throw new RuntimeException("Only tuple types are supported")
}
http://git-wip-us.apache.org/repos/asf/flink/blob/14134842/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
index 7a39da5..4095645 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamJoinOperator.scala
@@ -61,9 +61,12 @@ object StreamJoinOperator {
* The resulting incomplete join can be completed by JoinPredicate.equalTo()
* to define the second key.
*/
- def where(firstField: String, otherFields: String*) = {
- new JoinPredicate[I1, I2](op, new PojoKeySelector[I1](op.input1.getType(),
- (firstField +: otherFields): _*))
+ def where(firstField: String, otherFields: String*) =
+ op.input1.getType() match {
+ case ccInfo: CaseClassTypeInfo[I1] => new JoinPredicate[I1, I2](op,
+ new CaseClassKeySelector[I1](ccInfo, firstField +: otherFields.toArray: _*))
+ case _ => new JoinPredicate[I1, I2](op, new PojoKeySelector[I1](
+ op.input1.getType(), (firstField +: otherFields): _*))
}
/**
@@ -104,9 +107,13 @@ object StreamJoinOperator {
* (first, second)
* To define a custom wrapping, use JoinedStream.apply(...)
*/
- def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] = {
- finish(new PojoKeySelector[I2](op.input2.getType(), (firstField +: otherFields): _*))
- }
+ def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
+ op.input2.getType() match {
+ case ccInfo: CaseClassTypeInfo[I2] => finish(
+ new CaseClassKeySelector[I2](ccInfo, firstField +: otherFields.toArray: _*))
+ case _ => finish(new PojoKeySelector[I2](op.input2.getType(),
+ (firstField +: otherFields): _*))
+ }
/**
* Creates a temporal join transformation by defining the second join key.
http://git-wip-us.apache.org/repos/asf/flink/blob/14134842/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
index 8c763fc..11f042d 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -39,6 +39,7 @@ import scala.collection.JavaConversions._
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
import org.apache.flink.streaming.api.function.aggregation.SumFunction
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
class WindowedDataStream[T](javaStream: JavaWStream[T]) {
@@ -77,8 +78,14 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
*
*/
def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
- new WindowedDataStream[T](javaStream.groupBy(firstField +: otherFields.toArray: _*))
-
+ javaStream.getType() match {
+ case ccInfo: CaseClassTypeInfo[T] => new WindowedDataStream[T](javaStream.groupBy(
+ new CaseClassKeySelector[T](ccInfo, firstField +: otherFields.toArray: _*)))
+ case _ => new WindowedDataStream[T](javaStream.groupBy(
+ firstField +: otherFields.toArray: _*))
+ }
+
+
/**
* Groups the elements of the WindowedDataStream using the given
* KeySelector function. The window sizes (evictions) and slide sizes
http://git-wip-us.apache.org/repos/asf/flink/blob/14134842/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
index 53d1dea..e0d1155 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/typeutils/CaseClassTypeInfo.scala
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.api.scala.typeutils
import org.apache.flink.api.common.typeinfo.TypeInformation
[08/36] flink git commit: [streaming] Temporal join and cross rework
for consistence and extended features
Posted by mb...@apache.org.
[streaming] Temporal join and cross rework for consistence and extended features
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/652327be
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/652327be
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/652327be
Branch: refs/heads/release-0.8
Commit: 652327be3a1196fd98e18a777a24e36e6574da28
Parents: 1922dbd
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Dec 19 21:05:01 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:54:26 2015 +0100
----------------------------------------------------------------------
.../api/datastream/ConnectedDataStream.java | 55 +--------
.../api/datastream/StreamCrossOperator.java | 50 ++++++--
.../api/datastream/StreamJoinOperator.java | 123 +++++++++++++------
.../api/datastream/TemporalOperator.java | 103 ++++++++++++++++
.../api/datastream/WindowDBOperator.java | 103 ----------------
.../api/function/co/CrossWindowFunction.java | 21 ++--
.../api/function/co/JoinWindowFunction.java | 17 ++-
.../streaming/api/WindowCrossJoinTest.java | 2 +-
.../streaming/examples/join/WindowJoin.java | 3 +-
.../flink/api/scala/streaming/DataStream.scala | 6 +-
10 files changed, 255 insertions(+), 228 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index c663315..65a6c37 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -17,15 +17,10 @@
package org.apache.flink.streaming.api.datastream;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.api.JobGraphBuilder;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -43,7 +38,6 @@ import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.util.Collector;
/**
* The ConnectedDataStream represents a stream for two different data types. It
@@ -545,54 +539,7 @@ public class ConnectedDataStream<IN1, IN2> {
return invokable;
}
- protected <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCross(
- CrossFunction<IN1, IN2, OUT> crossFunction, long windowSize, long slideInterval,
- TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
-
- TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(CrossFunction.class,
- crossFunction.getClass(), 2, null, null);
-
- CrossWindowFunction<IN1, IN2, OUT> crossWindowFunction = new CrossWindowFunction<IN1, IN2, OUT>(
- clean(crossFunction));
-
- return addGeneralWindowCombine(crossWindowFunction, outTypeInfo, windowSize, slideInterval,
- timestamp1, timestamp2);
- }
-
- private static class CrossWindowFunction<IN1, IN2, OUT> implements
- CoWindowFunction<IN1, IN2, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- private CrossFunction<IN1, IN2, OUT> crossFunction;
-
- public CrossWindowFunction(CrossFunction<IN1, IN2, OUT> crossFunction) {
- this.crossFunction = crossFunction;
- }
-
- @Override
- public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out)
- throws Exception {
- for (IN1 firstValue : first) {
- for (IN2 secondValue : second) {
- out.collect(crossFunction.cross(firstValue, secondValue));
- }
- }
- }
- }
-
- protected SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> addGeneralWindowJoin(
- CoWindowFunction<IN1, IN2, Tuple2<IN1, IN2>> coWindowFunction, long windowSize,
- long slideInterval, TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
-
- TypeInformation<Tuple2<IN1, IN2>> outType = new TupleTypeInfo<Tuple2<IN1, IN2>>(
- getInputType1(), getInputType2());
-
- return addGeneralWindowCombine(coWindowFunction, outType, windowSize, slideInterval,
- timestamp1, timestamp2);
- }
-
- private <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
+ protected <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
CoWindowFunction<IN1, IN2, OUT> coWindowFunction, TypeInformation<OUT> outTypeInfo,
long windowSize, long slideInterval, TimeStamp<IN1> timestamp1,
TimeStamp<IN2> timestamp2) {
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
index f54f273..c6cba63 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
@@ -19,9 +19,16 @@
package org.apache.flink.streaming.api.datastream;
import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.operators.CrossOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
-public class StreamCrossOperator<I1, I2> extends WindowDBOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
+public class StreamCrossOperator<I1, I2> extends
+ TemporalOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
public StreamCrossOperator(DataStream<I1> input1, DataStream<I2> input2) {
super(input1, input2);
@@ -40,12 +47,23 @@ public class StreamCrossOperator<I1, I2> extends WindowDBOperator<I1, I2, Stream
this.op = operator;
}
+ public <F> F clean(F f) {
+ if (op.input1.getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
+ ClosureCleaner.clean(f, true);
+ }
+ ClosureCleaner.ensureSerializable(f);
+ return f;
+ }
+
/**
- * Finalizes a temporal Cross transformation by applying a {@link CrossFunction} to each pair of crossed elements.<br/>
- * Each CrossFunction call returns exactly one element.
+ * Finalizes a temporal Cross transformation by applying a
+ * {@link CrossFunction} to each pair of crossed elements.<br/>
+ * Each CrossFunction call returns exactly one element.
*
- * @param function The CrossFunction that is called for each pair of crossed elements.
- * @return An CrossOperator that represents the crossed result DataSet
+ * @param function
+ * The CrossFunction that is called for each pair of crossed
+ * elements.
+ * @return A CrossOperator that represents the crossed result DataStream
*
* @see CrossFunction
* @see DataSet
@@ -54,15 +72,29 @@ public class StreamCrossOperator<I1, I2> extends WindowDBOperator<I1, I2, Stream
return createCrossOperator(function);
}
+ /**
+ * Finalizes a temporal Cross transformation by emitting all pairs in a
+ * new Tuple2.
+ *
+ * @return A CrossOperator that represents the crossed result DataStream
+ */
+ public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> withDefault() {
+ return createCrossOperator(new CrossOperator.DefaultCrossFunction<I1, I2>());
+ }
+
protected <R> SingleOutputStreamOperator<R, ?> createCrossOperator(
CrossFunction<I1, I2, R> function) {
- return op.input1.connect(op.input2).addGeneralWindowCross(function, op.windowSize,
- op.slideInterval, op.timeStamp1, op.timeStamp2);
+ TypeInformation<R> outTypeInfo = TypeExtractor.getCrossReturnTypes(function,
+ op.input1.getType(), op.input2.getType());
- }
+ CrossWindowFunction<I1, I2, R> crossWindowFunction = new CrossWindowFunction<I1, I2, R>(
+ clean(function));
- // ----------------------------------------------------------------------------------------
+ return op.input1.connect(op.input2).addGeneralWindowCombine(crossWindowFunction,
+ outTypeInfo, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2);
+
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
index 89c80ab..3051587 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
@@ -18,15 +18,18 @@
package org.apache.flink.streaming.api.datastream;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
import org.apache.flink.streaming.util.keys.FieldsKeySelector;
import org.apache.flink.streaming.util.keys.PojoKeySelector;
public class StreamJoinOperator<I1, I2> extends
- WindowDBOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
+ TemporalOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
public StreamJoinOperator(DataStream<I1> input1, DataStream<I2> input2) {
super(input1, input2);
@@ -51,7 +54,7 @@ public class StreamJoinOperator<I1, I2> extends
* that should be used as join keys.<br/>
* <b>Note: Fields can only be selected as join keys on Tuple
* DataStreams.</b><br/>
- *
+ *
* @param fields
* The indexes of the other Tuple fields of the first join
* DataStreams that should be used as keys.
@@ -59,8 +62,8 @@ public class StreamJoinOperator<I1, I2> extends
* {@link JoinPredicate#equalTo} to continue the Join.
*/
public JoinPredicate<I1, I2> where(int... fields) {
- return new JoinPredicate<I1, I2>(op, FieldsKeySelector.getSelector(
- op.input1.getType(), fields));
+ return new JoinPredicate<I1, I2>(op, FieldsKeySelector.getSelector(op.input1.getType(),
+ fields));
}
/**
@@ -68,7 +71,7 @@ public class StreamJoinOperator<I1, I2> extends
* Defines the fields of the first join {@link DataStream} that should
* be used as grouping keys. Fields are the names of member fields of
* the underlying type of the data stream.
- *
+ *
* @param fields
* The fields of the first join DataStream that should be
* used as keys.
@@ -105,12 +108,13 @@ public class StreamJoinOperator<I1, I2> extends
* Intermediate step of a temporal Join transformation. <br/>
* To continue the Join transformation, select the join key of the second
* input {@link DataStream} by calling {@link JoinPredicate#equalTo}
- *
+ *
*/
public static class JoinPredicate<I1, I2> {
- private StreamJoinOperator<I1, I2> op;
- private final KeySelector<I1, ?> keys1;
+ public StreamJoinOperator<I1, I2> op;
+ public KeySelector<I1, ?> keys1;
+ public KeySelector<I2, ?> keys2;
private JoinPredicate(StreamJoinOperator<I1, I2> operator, KeySelector<I1, ?> keys1) {
this.op = operator;
@@ -124,37 +128,30 @@ public class StreamJoinOperator<I1, I2> extends
* <b>Note: Fields can only be selected as join keys on Tuple
* DataStreams.</b><br/>
*
- * The resulting operator wraps each pair of joining elements into a
- * {@link Tuple2}, with the element of the first input being the first
- * field of the tuple and the element of the second input being the
- * second field of the tuple.
- *
* @param fields
* The indexes of the Tuple fields of the second join
* DataStream that should be used as keys.
- * @return The joined data stream.
+ * @return An incomplete join. Call {@link FinalizeStreamJoin#with} or
+ * {@link FinalizeStreamJoin#withDefault} to complete
*/
- public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(int... fields) {
- return createJoinOperator(FieldsKeySelector.getSelector(op.input2.getType(),
- fields));
+ public FinalizeStreamJoin<I1, I2> equalTo(int... fields) {
+ keys2 = FieldsKeySelector.getSelector(op.input2.getType(), fields);
+ return new FinalizeStreamJoin<I1, I2>(this);
}
/**
* Continues a temporal Join transformation and defines the fields of
* the second join {@link DataStream} that should be used as join keys.<br/>
- *
- * The resulting operator wraps each pair of joining elements into a
- * {@link Tuple2}, with the element of the first input being the first
- * field of the tuple and the element of the second input being the
- * second field of the tuple.
- *
+ *
* @param fields
* The fields of the second join DataStream that should be
* used as keys.
- * @return The joined data stream.
+ * @return An incomplete join. Call {@link FinalizeStreamJoin#with} or
+ * {@link FinalizeStreamJoin#withDefault} to complete
*/
- public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(String... fields) {
- return createJoinOperator(new PojoKeySelector<I2>(op.input2.getType(), fields));
+ public FinalizeStreamJoin<I1, I2> equalTo(String... fields) {
+ this.keys2 = new PojoKeySelector<I2>(op.input2.getType(), fields);
+ return new FinalizeStreamJoin<I1, I2>(this);
}
/**
@@ -164,29 +161,75 @@ public class StreamJoinOperator<I1, I2> extends
* second DataStream and extracts a single key value on which the
* DataStream is joined. </br>
*
- * The resulting operator wraps each pair of joining elements into a
- * {@link Tuple2}, with the element of the first input being the first
- * field of the tuple and the element of the second input being the
- * second field of the tuple.
- *
* @param keySelector
* The KeySelector function which extracts the key values
* from the second DataStream on which it is joined.
+ * @return An incomplete join. Call {@link FinalizeStreamJoin#with} or
+ * {@link FinalizeStreamJoin#withDefault} to complete
+ */
+ public <K> FinalizeStreamJoin<I1, I2> equalTo(KeySelector<I2, K> keySelector) {
+ this.keys2 = keySelector;
+ return new FinalizeStreamJoin<I1, I2>(this);
+ }
+
+ }
+
+ public static class FinalizeStreamJoin<I1, I2> {
+ private final JoinPredicate<I1, I2> predicate;
+
+ private FinalizeStreamJoin(JoinPredicate<I1, I2> predicate) {
+ this.predicate = predicate;
+ }
+
+ /**
+ * Completes a stream join. </p> The resulting operator wraps each pair
+ * of joining elements into a {@link Tuple2}, with the element of the
+ * first input being the first field of the tuple and the element of the
+ * second input being the second field of the tuple.
+ *
+ * @return The joined data stream.
+ */
+ public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> withDefault() {
+ return createJoinOperator(new DefaultJoinFunction<I1, I2>());
+ }
+
+ /**
+ * Completes a stream join. </p> The resulting operator wraps each pair
+ * of joining elements using the user defined {@link JoinFunction}
+ *
* @return The joined data stream.
*/
- public <K> SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(
- KeySelector<I2, K> keySelector) {
- return createJoinOperator(keySelector);
+ public <OUT> SingleOutputStreamOperator<OUT, ?> with(JoinFunction<I1, I2, OUT> joinFunction) {
+ return createJoinOperator(joinFunction);
}
- protected SingleOutputStreamOperator<Tuple2<I1, I2>, ?> createJoinOperator(
- KeySelector<I2, ?> keys2) {
+ private <OUT> SingleOutputStreamOperator<OUT, ?> createJoinOperator(
+ JoinFunction<I1, I2, OUT> joinFunction) {
+
+ JoinWindowFunction<I1, I2, OUT> joinWindowFunction = new JoinWindowFunction<I1, I2, OUT>(
+ predicate.keys1, predicate.keys2, joinFunction);
+
+ StreamJoinOperator<I1, I2> op = predicate.op;
+
+ TypeInformation<OUT> outType = TypeExtractor.getJoinReturnTypes(joinFunction,
+ op.input1.getType(), op.input2.getType());
- JoinWindowFunction<I1, I2> joinWindowFunction = new JoinWindowFunction<I1, I2>(keys1,
- keys2);
- return op.input1.connect(op.input2).addGeneralWindowJoin(joinWindowFunction,
- op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2);
+ return op.input1.connect(op.input2).addGeneralWindowCombine(joinWindowFunction,
+ outType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2);
}
}
+ public static final class DefaultJoinFunction<T1, T2> implements
+ JoinFunction<T1, T2, Tuple2<T1, T2>> {
+
+ private static final long serialVersionUID = 1L;
+ private final Tuple2<T1, T2> outTuple = new Tuple2<T1, T2>();
+
+ @Override
+ public Tuple2<T1, T2> join(T1 first, T2 second) throws Exception {
+ outTuple.f0 = first;
+ outTuple.f1 = second;
+ return outTuple;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
new file mode 100644
index 0000000..cd8aabd
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+
+public abstract class TemporalOperator<I1, I2, OP> {
+
+ public final DataStream<I1> input1;
+ public final DataStream<I2> input2;
+
+ public long windowSize;
+ public long slideInterval;
+
+ public TimeStamp<I1> timeStamp1;
+ public TimeStamp<I2> timeStamp2;
+
+ public TemporalOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ if (input1 == null || input2 == null) {
+ throw new NullPointerException();
+ }
+ this.input1 = input1.copy();
+ this.input2 = input2.copy();
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public OP onWindow(long windowSize) {
+ return onWindow(windowSize, windowSize);
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param slideInterval
+ * The slide size of the window.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public OP onWindow(long windowSize, long slideInterval) {
+ return onWindow(windowSize, slideInterval, new DefaultTimeStamp<I1>(),
+ new DefaultTimeStamp<I2>());
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param slideInterval
+ * The slide size of the window.
+ * @param timeStamp1
+ * The timestamp used to extract time from the elements of the
+ * first data stream.
+ * @param timeStamp2
+ * The timestamp used to extract time from the elements of the
+ * second data stream.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public OP onWindow(long windowSize, long slideInterval, TimeStamp<I1> timeStamp1,
+ TimeStamp<I2> timeStamp2) {
+
+ this.windowSize = windowSize;
+ this.slideInterval = slideInterval;
+
+ this.timeStamp1 = timeStamp1;
+ this.timeStamp2 = timeStamp2;
+
+ return createNextWindowOperator();
+ }
+
+ protected abstract OP createNextWindowOperator();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java
deleted file mode 100644
index 4f6f0c1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-
-public abstract class WindowDBOperator<I1, I2, OP> {
-
- protected final DataStream<I1> input1;
- protected final DataStream<I2> input2;
-
- long windowSize;
- long slideInterval;
-
- TimeStamp<I1> timeStamp1;
- TimeStamp<I2> timeStamp2;
-
- public WindowDBOperator(DataStream<I1> input1, DataStream<I2> input2) {
- if (input1 == null || input2 == null) {
- throw new NullPointerException();
- }
- this.input1 = input1.copy();
- this.input2 = input2.copy();
- }
-
- /**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
- */
- public OP onWindow(long windowSize) {
- return onWindow(windowSize, windowSize);
- }
-
- /**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param slideInterval
- * The slide size of the window.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
- */
- public OP onWindow(long windowSize, long slideInterval) {
- return onWindow(windowSize, slideInterval, new DefaultTimeStamp<I1>(),
- new DefaultTimeStamp<I2>());
- }
-
- /**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param slideInterval
- * The slide size of the window.
- * @param timeStamp1
- * The timestamp used to extract time from the elements of the
- * first data stream.
- * @param timeStamp2
- * The timestamp used to extract time from the elements of the
- * second data stream.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
- */
- public OP onWindow(long windowSize, long slideInterval, TimeStamp<I1> timeStamp1,
- TimeStamp<I2> timeStamp2) {
-
- this.windowSize = windowSize;
- this.slideInterval = slideInterval;
-
- this.timeStamp1 = timeStamp1;
- this.timeStamp2 = timeStamp2;
-
- return createNextWindowOperator();
- }
-
- protected abstract OP createNextWindowOperator();
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
index f61738a..9cafcd1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/CrossWindowFunction.java
@@ -20,20 +20,25 @@ package org.apache.flink.streaming.api.function.co;
import java.util.List;
-import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.util.Collector;
-public class CrossWindowFunction<IN1, IN2> implements CoWindowFunction<IN1, IN2, Tuple2<IN1, IN2>> {
+public class CrossWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1, IN2, OUT> {
private static final long serialVersionUID = 1L;
- @Override
- public void coWindow(List<IN1> first, List<IN2> second, Collector<Tuple2<IN1, IN2>> out)
- throws Exception {
+ private CrossFunction<IN1, IN2, OUT> crossFunction;
+
+ public CrossWindowFunction(CrossFunction<IN1, IN2, OUT> crossFunction) {
+ this.crossFunction = crossFunction;
+ }
- for (IN1 item1 : first) {
- for (IN2 item2 : second) {
- out.collect(new Tuple2<IN1, IN2>(item1, item2));
+ @Override
+ public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
+ for (IN1 firstValue : first) {
+ for (IN2 secondValue : second) {
+ out.collect(crossFunction.cross(firstValue, secondValue));
}
}
}
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
index 53e2657..9f5cd5d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
@@ -20,31 +20,30 @@ package org.apache.flink.streaming.api.function.co;
import java.util.List;
+import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.util.Collector;
-public class JoinWindowFunction<IN1, IN2> implements CoWindowFunction<IN1, IN2, Tuple2<IN1, IN2>> {
+public class JoinWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1, IN2, OUT> {
private static final long serialVersionUID = 1L;
private KeySelector<IN1, ?> keySelector1;
private KeySelector<IN2, ?> keySelector2;
+ private JoinFunction<IN1, IN2, OUT> joinFunction;
- public JoinWindowFunction() {
- }
-
- public JoinWindowFunction(KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2) {
+ public JoinWindowFunction(KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2,
+ JoinFunction<IN1, IN2, OUT> joinFunction) {
this.keySelector1 = keySelector1;
this.keySelector2 = keySelector2;
+ this.joinFunction = joinFunction;
}
@Override
- public void coWindow(List<IN1> first, List<IN2> second, Collector<Tuple2<IN1, IN2>> out)
- throws Exception {
+ public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
for (IN1 item1 : first) {
for (IN2 item2 : second) {
if (keySelector1.getKey(item1).equals(keySelector2.getKey(item2))) {
- out.collect(new Tuple2<IN1, IN2>(item1, item2));
+ out.collect(joinFunction.join(item1, item2));
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
index d8cdfa5..ffdfbc3 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -96,7 +96,7 @@ public class WindowCrossJoinTest implements Serializable {
DataStream<Integer> inStream2 = env.fromCollection(in2);
inStream1.join(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
- .where(0).equalTo(0).addSink(new JoinResultSink());
+ .where(0).equalTo(0).withDefault().addSink(new JoinResultSink());
inStream1.cross(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
.with(new CrossFunction<Tuple2<Integer,String>, Integer, Tuple2<Tuple2<Integer,String>, Integer>>() {
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
index 93df823..2586e3c 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
@@ -67,7 +67,8 @@ public class WindowJoin {
.join(salaries)
.onWindow(1000)
.where(0)
- .equalTo(0);
+ .equalTo(0)
+ .withDefault();
// emit result
if (fileOutput) {
http://git-wip-us.apache.org/repos/asf/flink/blob/652327be/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 5f46c84..a117412 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -284,7 +284,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* received records.
*
*/
- def count: DataStream[java.lang.Long] = new DataStream[java.lang.Long](javaStream.count())
+ def count: DataStream[Long] = new DataStream[java.lang.Long](javaStream.count()).asInstanceOf[DataStream[Long]]
/**
* Creates a new DataStream by applying the given function to every element of this DataStream.
@@ -445,14 +445,14 @@ class DataStream[T](javaStream: JavaStream[T]) {
* Creates a new SplitDataStream that contains only the elements satisfying the
* given output selector predicate.
*/
- def split(fun: T => TraversableOnce[String]): SplitDataStream[T] = {
+ def split(fun: T => String): SplitDataStream[T] = {
if (fun == null) {
throw new NullPointerException("OutputSelector must not be null.")
}
val selector = new OutputSelector[T] {
val cleanFun = clean(fun)
def select(in: T): java.lang.Iterable[String] = {
- asJavaIterable(cleanFun(in).toIterable)
+ List(cleanFun(in))
}
}
split(selector)
[05/36] flink git commit: [scala] [streaming] Windowing functionality
added to scala api
Posted by mb...@apache.org.
[scala] [streaming] Windowing functionality added to scala api
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e0ab4898
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e0ab4898
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e0ab4898
Branch: refs/heads/release-0.8
Commit: e0ab48988d22ba2a38a8e6e43d9ede6215fdddb5
Parents: d1b53b1
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Dec 15 16:21:00 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:52:46 2015 +0100
----------------------------------------------------------------------
.../api/datastream/WindowedDataStream.java | 54 +++++
.../flink/api/scala/streaming/DataStream.scala | 27 +++
.../streaming/StreamExecutionEnvironment.scala | 14 +-
.../scala/streaming/WindowedDataStream.scala | 232 +++++++++++++++++++
4 files changed, 326 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e0ab4898/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index cb9cd04..287f29d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -260,6 +260,30 @@ public class WindowedDataStream<OUT> {
}
/**
+ * Applies a reduceGroup transformation on the windowed data stream by
+ * reducing the current window at every trigger. In contrast with the
+ * standard binary reducer, with reduceGroup the user can access all
+ * elements of the window at the same time through the iterable interface.
+ * The user can also extend the {@link RichGroupReduceFunction} to gain
+ * access to other features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ * </br> </br> This version of reduceGroup uses user supplied
+ * typeinformation for serializaton. Use this only when the system is unable
+ * to detect type information using:
+ * {@link #reduceGroup(GroupReduceFunction)}
+ *
+ * @param reduceFunction
+ * The reduce function that will be applied to the windows.
+ * @return The transformed DataStream
+ */
+ public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
+ GroupReduceFunction<OUT, R> reduceFunction, TypeInformation<R> outType) {
+
+ return dataStream.transform("NextGenWindowReduce", outType,
+ getReduceGroupInvokable(reduceFunction));
+ }
+
+ /**
* Applies an aggregation that sums every window of the data stream at the
* given position.
*
@@ -335,6 +359,19 @@ public class WindowedDataStream<OUT> {
/**
* Applies an aggregation that gives the minimum element of every window of
* the data stream by the given position. If more elements have the same
+ * minimum value the operator returns the first element by default.
+ *
+ * @param positionToMinBy
+ * The position to minimize by
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(String positionToMinBy) {
+ return this.minBy(positionToMinBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum element of every window of
+ * the data stream by the given position. If more elements have the same
* minimum value the operator returns either the first or last one depending
* on the parameter setting.
*
@@ -418,6 +455,19 @@ public class WindowedDataStream<OUT> {
/**
* Applies an aggregation that gives the maximum element of every window of
* the data stream by the given position. If more elements have the same
+ * maximum value the operator returns the first by default.
+ *
+ * @param positionToMaxBy
+ * The position to maximize by
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(String positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum element of every window of
+ * the data stream by the given position. If more elements have the same
* maximum value the operator returns either the first or last one depending
* on the parameter setting.
*
@@ -598,6 +648,10 @@ public class WindowedDataStream<OUT> {
return dataStream.getType();
}
+ public DataStream<OUT> getDataStream() {
+ return dataStream;
+ }
+
protected WindowedDataStream<OUT> copy() {
return new WindowedDataStream<OUT>(this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0ab4898/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index e96f5eb..69b8359 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -38,6 +38,10 @@ import org.apache.flink.api.common.functions.ReduceFunction
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.common.functions.FilterFunction
import org.apache.flink.streaming.api.function.sink.SinkFunction
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy
+import scala.collection.JavaConversions._
class DataStream[T](javaStream: JavaStream[T]) {
@@ -397,6 +401,29 @@ class DataStream[T](javaStream: JavaStream[T]) {
}
/**
+ * Create a WindowedDataStream that can be used to apply
+ * transformation like .reduce(...) or aggregations on
+ * preset chunks(windows) of the data stream. To define the windows one or
+ * more WindowingHelper-s such as Time, Count and
+ * Delta can be used.</br></br> When applied to a grouped data
+ * stream, the windows (evictions) and slide sizes (triggers) will be
+ * computed on a per group basis. </br></br> For more advanced control over
+ * the trigger and eviction policies please use to
+ * window(List(triggers), List(evicters))
+ */
+ def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] = new WindowedDataStream[T](javaStream.window(windowingHelper: _*))
+
+ /**
+ * Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
+ * Windowing can be used to apply transformation like .reduce(...) or aggregations on
+ * preset chunks(windows) of the data stream.</br></br>For most common
+ * use-cases please refer to window(WindowingHelper[_]*)
+ *
+ */
+ def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]): WindowedDataStream[T] = new WindowedDataStream[T](javaStream.window(triggers, evicters))
+
+ /**
+ * >>>>>>> 12178aa... [scala] [streaming] Windowing functionality added to scala api
* Writes a DataStream to the standard output stream (stdout). For each
* element of the DataStream the result of .toString is
* written.
http://git-wip-us.apache.org/repos/asf/flink/blob/e0ab4898/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index e4a7b48..dadfde2 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -18,6 +18,7 @@
package org.apache.flink.api.scala.streaming
+import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.environment.{ StreamExecutionEnvironment => JavaEnv }
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.commons.lang.Validate
@@ -26,6 +27,8 @@ import org.apache.flink.streaming.api.datastream.DataStreamSource
import org.apache.flink.streaming.api.invokable.SourceInvokable
import org.apache.flink.streaming.api.function.source.FromElementsFunction
import org.apache.flink.streaming.api.function.source.SourceFunction
+import scala.collection.JavaConversions._
+import org.apache.flink.util.Collector
class StreamExecutionEnvironment(javaEnv: JavaEnv) {
@@ -113,7 +116,16 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
* Creates a new DataStream that contains a sequence of numbers.
*
*/
- def generateSequence(from: Long, to: Long): DataStream[java.lang.Long] = new DataStream(javaEnv.generateSequence(from, to))
+ def generateSequence(from: Long, to: Long): DataStream[Long] = {
+ val source = new SourceFunction[Long] {
+ override def invoke(out: Collector[Long]) = {
+ for (i <- from.to(to)) {
+ out.collect(i)
+ }
+ }
+ }
+ addSource(source)
+ }
/**
* Creates a DataStream that contains the given elements. The elements must all be of the
http://git-wip-us.apache.org/repos/asf/flink/blob/e0ab4898/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
new file mode 100644
index 0000000..ff89a47
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import scala.reflect.ClassTag
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.streaming.api.invokable.operator.MapInvokable
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator
+import org.apache.flink.util.Collector
+import org.apache.flink.api.common.functions.FlatMapFunction
+import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
+import org.apache.flink.api.common.functions.GroupReduceFunction
+import org.apache.flink.streaming.api.invokable.StreamInvokable
+import scala.collection.JavaConversions._
+
+class WindowedDataStream[T](javaStream: JavaWStream[T]) {
+
+ private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
+ ClosureCleaner.clean(f, checkSerializable)
+ f
+ }
+
+ /**
+ * Defines the slide size (trigger frequency) for the windowed data stream.
+ * This controls how often the user defined function will be triggered on
+ * the window.
+ */
+ def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] = new WindowedDataStream[T](javaStream.every(windowingHelper: _*))
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * field positions. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy(fields: Int*): WindowedDataStream[T] =
+ new WindowedDataStream[T](javaStream.groupBy(fields: _*))
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * field expressions. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
+ new WindowedDataStream[T](javaStream.groupBy(firstField +: otherFields.toArray: _*))
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * KeySelector function. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy[K: TypeInformation](fun: T => K): WindowedDataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ new WindowedDataStream[T](javaStream.groupBy(keyExtractor))
+ }
+
+ /**
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.
+ *
+ */
+ def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
+ if (reducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ new DataStream[T](javaStream.reduce(reducer))
+ }
+
+ /**
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.
+ *
+ */
+ def reduce(fun: (T, T) => T): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ val reducer = new ReduceFunction[T] {
+ val cleanFun = clean(fun)
+ def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by reducing
+ * the current window at every trigger. In contrast with the simple binary reduce operator, groupReduce exposes the whole window through the Iterable interface.
+ * </br>
+ * </br>
+ * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+ */
+ def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]): DataStream[R] = {
+ if (reducer == null) {
+ throw new NullPointerException("GroupReduce function must not be null.")
+ }
+ new DataStream[R](javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]]))
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by reducing
+ * the current window at every trigger. In contrast with the simple binary reduce operator, groupReduce exposes the whole window through the Iterable interface.
+ * </br>
+ * </br>
+ * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+ */
+ def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("GroupReduce function must not be null.")
+ }
+ val reducer = new GroupReduceFunction[T, R] {
+ val cleanFun = clean(fun)
+ def reduce(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
+ }
+ reduceGroup(reducer)
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum of the elements in the window at
+ * the given position.
+ *
+ */
+ def max(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.max(field))
+ case field: String => return new DataStream[T](javaStream.max(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum of the elements in the window at
+ * the given position.
+ *
+ */
+ def min(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.min(field))
+ case field: String => return new DataStream[T](javaStream.min(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ /**
+ * Applies an aggregation that sums the elements in the window at the given position.
+ *
+ */
+ def sum(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.sum(field))
+ case field: String => return new DataStream[T](javaStream.sum(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum element of the window by
+ * the given position. When equality, returns the first.
+ *
+ */
+ def maxBy(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.maxBy(field))
+ case field: String => return new DataStream[T](javaStream.maxBy(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum element of the window by
+ * the given position. When equality, returns the first.
+ *
+ */
+ def minBy(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.minBy(field))
+ case field: String => return new DataStream[T](javaStream.minBy(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum element of the window by
+ * the given position. When equality, the user can set to get the first or last element with the minimal value.
+ *
+ */
+ def minBy(field: Any, first: Boolean): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.minBy(field, first))
+ case field: String => return new DataStream[T](javaStream.minBy(field, first))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum element of the window by
+ * the given position. When equality, the user can set to get the first or last element with the maximal value.
+ *
+ */
+ def maxBy(field: Any, first: Boolean): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.maxBy(field, first))
+ case field: String => return new DataStream[T](javaStream.maxBy(field, first))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+}
\ No newline at end of file
[07/36] flink git commit: [scala] [streaming] Added support for
iterative streams for scala api
Posted by mb...@apache.org.
[scala] [streaming] Added support for iterative streams for scala api
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1922dbdb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1922dbdb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1922dbdb
Branch: refs/heads/release-0.8
Commit: 1922dbdb78019136310f0214395a9b0de27bbc49
Parents: a4c4f26
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Dec 18 22:46:00 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:53:54 2015 +0100
----------------------------------------------------------------------
docs/streaming_guide.md | 4 +-
.../streaming/api/datastream/DataStream.java | 38 ++++++++--
.../api/datastream/IterativeDataStream.java | 18 +----
.../apache/flink/streaming/api/IterateTest.java | 2 +-
.../examples/iteration/IterateExample.java | 7 +-
.../flink/api/scala/streaming/DataStream.scala | 74 ++++++++------------
.../streaming/StreamExecutionEnvironment.scala | 9 +--
7 files changed, 73 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1922dbdb/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index c51afbc..bea0907 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -509,7 +509,7 @@ Unlike in the core API the user does not define the maximum number of iterations
To start an iterative part of the program the user defines the iteration starting point:
~~~java
-IterativeDataStream<Integer> iteration = source.iterate();
+IterativeDataStream<Integer> iteration = source.iterate(maxWaitTimeMillis);
~~~
The operator applied on the iteration starting point is the head of the iteration, where data is fed back from the iteration tail.
@@ -529,7 +529,7 @@ iteration.closeWith(tailOperator.select("iterate"));
In these case all output directed to the “iterate” edge would be fed back to the iteration head.
Because iterative streaming programs do not have a set number of iterations for each data element, the streaming program has no information on the end of its input. From this it follows that iterative streaming programs run until the user manually stops the program. While this is acceptable under normal circumstances a method is provided to allow iterative programs to shut down automatically if no input received by the iteration head for a predefined number of milliseconds.
-To use this function the user needs to call, the `iteration.setMaxWaitTime(millis)` to control the max wait time.
+To use this functionality the user needs to add the maxWaitTimeMillis parameter to the `dataStream.iterate(…)` call to control the max wait time.
### Rich functions
The usage of rich functions are essentially the same as in the core Flink API. All transformations that take as argument a user-defined function can instead take a rich function as argument:
http://git-wip-us.apache.org/repos/asf/flink/blob/1922dbdb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 3a3cdc4..a236312 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -383,14 +383,44 @@ public class DataStream<OUT> {
* the iteration head.
* <p>
* By default a DataStream with iteration will never terminate, but the user
- * can use the {@link IterativeDataStream#setMaxWaitTime} call to set a max
- * waiting time for the iteration head. If no data received in the set time,
- * the stream terminates.
+ * can use the the maxWaitTime parameter to set a max waiting time for the
+ * iteration head. If no data received in the set time, the stream
+ * terminates.
*
* @return The iterative data stream created.
*/
public IterativeDataStream<OUT> iterate() {
- return new IterativeDataStream<OUT>(this);
+ return new IterativeDataStream<OUT>(this, 0);
+ }
+
+ /**
+ * Initiates an iterative part of the program that feeds back data streams.
+ * The iterative part needs to be closed by calling
+ * {@link IterativeDataStream#closeWith(DataStream)}. The transformation of
+ * this IterativeDataStream will be the iteration head. The data stream
+ * given to the {@link IterativeDataStream#closeWith(DataStream)} method is
+ * the data stream that will be fed back and used as the input for the
+ * iteration head. A common usage pattern for streaming iterations is to use
+ * output splitting to send a part of the closing data stream to the head.
+ * Refer to {@link SingleOutputStreamOperator#split(OutputSelector)} for
+ * more information.
+ * <p>
+ * The iteration edge will be partitioned the same way as the first input of
+ * the iteration head.
+ * <p>
+ * By default a DataStream with iteration will never terminate, but the user
+ * can use the the maxWaitTime parameter to set a max waiting time for the
+ * iteration head. If no data received in the set time, the stream
+ * terminates.
+ *
+ * @param maxWaitTimeMillis
+ * Number of milliseconds to wait between inputs before shutting
+ * down
+ *
+ * @return The iterative data stream created.
+ */
+ public IterativeDataStream<OUT> iterate(long maxWaitTimeMillis) {
+ return new IterativeDataStream<OUT>(this, maxWaitTimeMillis);
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/1922dbdb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index d8497ae..78518c0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -31,12 +31,12 @@ public class IterativeDataStream<IN> extends
protected Integer iterationID;
protected long waitTime;
- protected IterativeDataStream(DataStream<IN> dataStream) {
+ protected IterativeDataStream(DataStream<IN> dataStream, long maxWaitTime) {
super(dataStream);
setBufferTimeout(dataStream.environment.getBufferTimeout());
iterationID = iterationCount;
iterationCount++;
- waitTime = 0;
+ waitTime = maxWaitTime;
}
protected IterativeDataStream(DataStream<IN> dataStream, Integer iterationID, long waitTime) {
@@ -71,20 +71,6 @@ public class IterativeDataStream<IN> extends
return iterationTail;
}
- /**
- * Sets the max waiting time for the next record before shutting down the
- * stream. If not set, then the user needs to manually kill the process to
- * stop.
- *
- * @param waitTimeMillis
- * Max waiting time in milliseconds
- * @return The modified DataStream.
- */
- public IterativeDataStream<IN> setMaxWaitTime(long waitTimeMillis) {
- this.waitTime = waitTimeMillis;
- return this;
- }
-
@Override
protected IterativeDataStream<IN> copy() {
return new IterativeDataStream<IN>(this, iterationID, waitTime);
http://git-wip-us.apache.org/repos/asf/flink/blob/1922dbdb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
index 5d910ba..712269c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/IterateTest.java
@@ -79,7 +79,7 @@ public class IterateTest {
DataStream<Boolean> source = env.fromElements(false, false, false);
- IterativeDataStream<Boolean> iteration = source.iterate().setMaxWaitTime(3000);
+ IterativeDataStream<Boolean> iteration = source.iterate(3000);
DataStream<Boolean> increment = iteration.flatMap(new IterationHead()).flatMap(
new IterationTail());
http://git-wip-us.apache.org/repos/asf/flink/blob/1922dbdb/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
index 4a51c41..a5dc68a 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/iteration/IterateExample.java
@@ -70,12 +70,9 @@ public class IterateExample {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment()
.setBufferTimeout(1);
- // create an iterative data stream from the input
+ // create an iterative data stream from the input with 5 second timeout
IterativeDataStream<Tuple2<Double, Integer>> it = env.fromCollection(input).shuffle()
- .iterate();
-
- // trigger iteration termination if no new data received for 5 seconds
- it.setMaxWaitTime(5000);
+ .iterate(5000);
// apply the step function to add new random value to the tuple and to
// increment the counter and split the output with the output selector
http://git-wip-us.apache.org/repos/asf/flink/blob/1922dbdb/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index f0932d8..5f46c84 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -68,7 +68,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
/**
* Gets the underlying java DataStream object.
*/
- private[flink] def getJavaStream: JavaStream[T] = javaStream
+ def getJavaStream: JavaStream[T] = javaStream
/**
* Sets the degree of parallelism of this operation. This must be greater than 1.
@@ -200,6 +200,32 @@ class DataStream[T](javaStream: JavaStream[T]) {
def distribute: DataStream[T] = new DataStream[T](javaStream.distribute())
/**
+ * Initiates an iterative part of the program that creates a loop by feeding
+ * back data streams. To create a streaming iteration the user needs to define
+ * a transformation that creates two DataStreams.The first one one is the output
+ * that will be fed back to the start of the iteration and the second is the output
+ * stream of the iterative part.
+ * <p>
+ * stepfunction: initialStream => (feedback, output)
+ * <p>
+ * A common pattern is to use output splitting to create feedback and output DataStream.
+ * Please refer to the .split(...) method of the DataStream
+ * <p>
+ * By default a DataStream with iteration will never terminate, but the user
+ * can use the maxWaitTime parameter to set a max waiting time for the iteration head.
+ * If no data received in the set time the stream terminates.
+ *
+ *
+ */
+ def iterate(stepFunction: DataStream[T] => (DataStream[T], DataStream[T]), maxWaitTimeMillis: Long = 0): DataStream[T] = {
+ val iterativeStream = javaStream.iterate(maxWaitTimeMillis)
+
+ val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
+ iterativeStream.closeWith(feedback.getJavaStream)
+ new DataStream[T](output.getJavaStream)
+ }
+
+ /**
* Applies an aggregation that that gives the current maximum of the data stream at
* the given position.
*
@@ -232,33 +258,11 @@ class DataStream[T](javaStream: JavaStream[T]) {
}
/**
- * Applies an aggregation that that gives the current maximum element of the data stream by
- * the given position. When equality, returns the first.
- *
- */
- def maxBy(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.maxBy(field))
- case field: String => return new DataStream[T](javaStream.maxBy(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
-
- /**
- * Applies an aggregation that that gives the current minimum element of the data stream by
- * the given position. When equality, returns the first.
- *
- */
- def minBy(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.minBy(field))
- case field: String => return new DataStream[T](javaStream.minBy(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
-
- /**
* Applies an aggregation that that gives the current minimum element of the data stream by
* the given position. When equality, the user can set to get the first or last element with the minimal value.
*
*/
- def minBy(field: Any, first: Boolean): DataStream[T] = field match {
+ def minBy(field: Any, first: Boolean = true): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.minBy(field, first))
case field: String => return new DataStream[T](javaStream.minBy(field, first))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
@@ -269,7 +273,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* the given position. When equality, the user can set to get the first or last element with the maximal value.
*
*/
- def maxBy(field: Any, first: Boolean): DataStream[T] = field match {
+ def maxBy(field: Any, first: Boolean = true): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.maxBy(field, first))
case field: String => return new DataStream[T](javaStream.maxBy(field, first))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
@@ -469,15 +473,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* is written.
*
*/
- def writeAsText(path: String, millis: Long): DataStream[T] = new DataStream[T](javaStream.writeAsText(path, millis))
-
- /**
- * Writes a DataStream to the file specified by path in text format.
- * For every element of the DataStream the result of .toString
- * is written.
- *
- */
- def writeAsText(path: String): DataStream[T] = new DataStream[T](javaStream.writeAsText(path))
+ def writeAsText(path: String, millis: Long = 0): DataStream[T] = new DataStream[T](javaStream.writeAsText(path, millis))
/**
* Writes a DataStream to the file specified by path in text format. The
@@ -486,15 +482,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
* is written.
*
*/
- def writeAsCsv(path: String, millis: Long): DataStream[T] = new DataStream[T](javaStream.writeAsCsv(path, millis))
-
- /**
- * Writes a DataStream to the file specified by path in text format.
- * For every element of the DataStream the result of .toString
- * is written.
- *
- */
- def writeAsCsv(path: String): DataStream[T] = new DataStream[T](javaStream.writeAsCsv(path))
+ def writeAsCsv(path: String, millis: Long = 0): DataStream[T] = new DataStream[T](javaStream.writeAsCsv(path, millis))
/**
* Adds the given sink to this DataStream. Only streams with sinks added
http://git-wip-us.apache.org/repos/asf/flink/blob/1922dbdb/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index dadfde2..340ecc1 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -117,14 +117,7 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*
*/
def generateSequence(from: Long, to: Long): DataStream[Long] = {
- val source = new SourceFunction[Long] {
- override def invoke(out: Collector[Long]) = {
- for (i <- from.to(to)) {
- out.collect(i)
- }
- }
- }
- addSource(source)
+ new DataStream[java.lang.Long](javaEnv.generateSequence(from, to)).asInstanceOf[DataStream[Long]]
}
/**
[32/36] flink git commit: [streaming] Updated streaming guide for
recent connector and data source changes
Posted by mb...@apache.org.
[streaming] Updated streaming guide for recent connector and data source changes
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/acd2d601
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/acd2d601
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/acd2d601
Branch: refs/heads/release-0.8
Commit: acd2d601e89e336c17c87bcb51461be0a4db5340
Parents: fcd28fc
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Jan 5 18:26:18 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 00:22:58 2015 +0100
----------------------------------------------------------------------
docs/streaming_guide.md | 309 +++----------------
.../api/scala/StreamExecutionEnvironment.scala | 6 +-
2 files changed, 56 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/acd2d601/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index bea0907..c7e7060 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -170,7 +170,9 @@ Usage: `operator.setParallelism(1)`
### Sources
-The user can connect to data streams by the different implementations of `SourceFunction` using `StreamExecutionEnvironment.addSource(SourceFunction)`. In contrast with other operators, DataStreamSources have a default operator parallelism of 1 which can be increased using the `.setParallelism(dop)` method as we will see later at the operator settings.
+The user can connect to data streams by the different implementations of `SourceFunction` using `StreamExecutionEnvironment.addSource(SourceFunction)`. In contrast with other operators, DataStreamSources have a default operator parallelism of 1.
+
+To create parallel sources the users source function needs to implement `ParallelSourceFunction` or extend `RichParallelSourceFunction` in which cases the source will have the parallelism of the environment. The degree of parallelism for ParallelSourceFunctions can be changed afterwards using `source.setParallelism(int dop)`.
There are several predefined ones similar to the ones of the batch API and some streaming specific ones like:
@@ -617,7 +619,7 @@ Stream connectors
Connectors provide an interface for accessing data from various third party sources (message queues). Currently four connectors are natively supported, namely [Apache Kafka](https://kafka.apache.org/), [RabbitMQ](http://www.rabbitmq.com/), [Apache Flume](https://flume.apache.org/index.html) and [Twitter Streaming API](https://dev.twitter.com/docs/streaming-apis).
-Typically the connector packages consist of an abstract source and sink (with the exception of Twitter where only a source is provided). The burden of the user is to implement a subclass of these abstract classes specifying a serializer and a deserializer function.
+Typically the connector packages consist of a source and sink class (with the exception of Twitter where only a source is provided). To use these sources the user needs to pass Serialization/Deserialization schemas for the connectors for the desired types. (Or use some predefined ones)
To run an application using one of these connectors usually additional third party components are required to be installed and launched, e.g. the servers for the message queues. Further instructions for these can be found in the corresponding subsections. [Docker containers](#docker-containers-for-connectors) are also provided encapsulating these services to aid users getting started with connectors.
@@ -631,106 +633,40 @@ This connector provides access to data streams from [Apache Kafka](https://kafka
* If the Kafka zookeeper and server are running on a remote machine then in the config/server.properties file the advertised.host.name must be set to the machine's IP address.
#### Kafka Source
-An abstract class providing an interface for receiving data from Kafka. By implementing the user must:
+A class providing an interface for receiving data from Kafka.
- * Write a constructor calling the constructor of the abstract class,
- * Write a deserializer function which processes the data coming from Kafka,
- * Stop the source manually when necessary with one of the close functions.
+The followings have to be provided for the `KafkaSource(..)` constructor in order:
-The implemented class must extend `KafkaSource`, for example: `KafkaSource<String>`.
+1. The hostname
+2. The group name
+3. The topic name
+4. The parallelism
+5. Deserialisation schema
-##### Constructor
-An example of an implementation of a constructor:
-~~~java
-public MyKafkaSource(String zkQuorum, String groupId, String topicId, int numThreads) {
- super(zkQuorum, groupId, topicId, numThreads);
-}
-~~~
-
-##### Deserializer
-An example of an implementation of a deserializer:
+Example:
~~~java
-@Override
-public String deserialize(byte[] msg) {
- String s = new String(msg);
- if(s.equals("q")){
- closeWithoutSend();
- }
- return new String(s);
-}
+DataStream<String> stream = env
+ .addSource(new KafkaSource<String>("localhost:2181", "group", "test",new SimpleStringSchema()))
+ .print();
~~~
-The source closes when it receives the String `"q"`.
-
-###### Close<a name="kafka_source_close"></a>
-Two types of close functions are available, namely `closeWithoutSend()` and `sendAndClose()`. The former closes the connection immediately and no further data will be sent, while the latter closes the connection only when the next message is sent after this call.
-
-In the example provided `closeWithoutSend()` is used because here the String `"q"` is meta-message indicating the end of the stream and there is no need to forward it.
-
#### Kafka Sink
-An abstract class providing an interface for sending data to Kafka. By implementing the user must:
-
- * Write a constructor calling the constructor of the abstract class,
- * Write a serializer function to send data in the desired form to Kafka,
- * Stop the sink manually when necessary with one of the close functions.
-
-The implemented class must extend `KafkaSink`, for example `KafkaSink<String, String>`.
-
-##### Constructor
-An example of an implementation of a constructor:
-
-~~~java
-public MyKafkaSink(String topicId, String brokerAddr) {
- super(topicId, brokerAddr);
-}
-~~~
-
-##### Serializer
-An example of an implementation of a serializer:
-
-~~~java
-@Override
-public String serialize(String tuple) {
- if(tuple.equals("q")){
- sendAndClose();
- }
- return tuple;
-}
-~~~
+A class providing an interface for sending data to Kafka.
-##### Close
-The API provided is the [same](#kafka_source_close) as the one for `KafkaSource`.
+The followings have to be provided for the `KafkaSink()` constructor in order:
-#### Building A Topology
-To use a Kafka connector as a source in Flink call the `addSource()` function with a new instance of the class which extends `KafkaSource` as parameter:
-
-~~~java
-DataStream<String> stream1 = env.
- addSource(new MyKafkaSource("localhost:2181", "group", "test", 1), SOURCE_PARALELISM)
- .print();
-~~~
-
-The followings have to be provided for the `MyKafkaSource()` constructor in order:
-
-1. The hostname
-2. The group name
-3. The topic name
-4. The parallelism
+1. The topic name
+2. The hostname
+3. Serialisation schema
-Similarly to use a Kafka connector as a sink in Flink call the `addSink()` function with a new instance of the class which extends `KafkaSink`:
+Example:
~~~java
-DataStream<String> stream2 = env
- .addSource(new MySource())
- .addSink(new MyKafkaSink("test", "localhost:9092"));
+stream.addSink(new KafkaSink<String, String>("test", "localhost:9092", new SimpleStringSchema()));
~~~
-The followings have to be provided for the `MyKafkaSink()` constructor in order:
-
-1. The topic name
-2. The hostname
More about Kafka can be found [here](https://kafka.apache.org/documentation.html).
@@ -741,114 +677,40 @@ More about Kafka can be found [here](https://kafka.apache.org/documentation.html
This connector provides access to datastreams from [Apache Flume](http://flume.apache.org/).
#### Installing Apache Flume
-[Download](http://flume.apache.org/download.html) Apache Flume. A configuration file is required for starting agents in Flume. A configuration file for running the example can be found [here](#config_file).
+[Download](http://flume.apache.org/download.html) Apache Flume. A configuration file is required for starting agents in Flume. A configuration file for running the example can be found [here](#config_file).
#### Flume Source
-An abstract class providing an interface for receiving data from Flume. By implementing the user must:
+A class providing an interface for receiving data from Flume.
- * Write a constructor calling the constructor of the abstract class,
- * Write a deserializer function which processes the data coming from Flume,
- * Stop the source manually when necessary with one of the close functions.
+The followings have to be provided for the `FlumeSource(…)` constructor in order:
-The implemented class must extend `FlumeSource` for example: `FlumeSource<String>`
-
-##### Constructor
-An example of an implementation of a constructor:
-
-~~~java
-MyFlumeSource(String host, int port) {
- super(host, port);
-}
-~~~
+1. The hostname
+2. The port number
+3. Deserialisation schema
-##### Deserializer
-An example of an implementation of a deserializer:
+Example:
~~~java
-@Override
-public String deserialize(byte[] msg) {
- String s = (String) SerializationUtils.deserialize(msg);
- String out = s;
- if (s.equals("q")) {
- closeWithoutSend();
- }
- return out;
-}
+DataStream<String> stream = env
+ .addSource(new FlumeSource<String>("localhost", 41414, new SimpleStringSchema()))
+ .print();
~~~
-The source closes when it receives the String `"q"`.
-
-##### Close<a name="flume_source_close"></a>
-Two types of close functions are available, namely `closeWithoutSend()` and `sendAndClose()`.The former closes the connection immediately and no further data will be sent, while the latter closes the connection only when the next message is sent after this call.
-
-In the example `closeWithoutSend()` is used because here the String `"q"` is meta-message indicating the end of the stream and there is no need to forward it.
-
#### Flume Sink
-An abstract class providing an interface for sending data to Flume. By implementing the user must:
-
-* Write a constructor calling the constructor of the abstract class,
-* Write a serializer function to send data in the desired form to Flume,
-* Stop the sink manually when necessary with one of the close functions.
+A class providing an interface for sending data to Flume.
-The implemented class must extend `FlumeSink`, for example `FlumeSink<String, String>`.
-
-##### Constructor
-An example of an implementation of a constructor:
-
-~~~java
-public MyFlumeSink(String host, int port) {
- super(host, port);
-}
-~~~
-
-##### Serializer
-An example of an implementation of a serializer.
-
-~~~java
-@Override
-public byte[] serialize(String tuple) {
- if (tuple.equals("q")) {
- try {
- sendAndClose();
- } catch (Exception e) {
- new RuntimeException("Error while closing Flume connection with " + port + " at "
- + host, e);
- }
- }
- return SerializationUtils.serialize(tuple);
-}
-~~~
-
-##### Close
-The API provided is the [same](#flume_source_close) as the one for `FlumeSource`.
-
-#### Building A Topology
-To use a Flume connector as a source in Flink call the `addSource()` function with a new instance of the class which extends `FlumeSource` as parameter:
-
-~~~java
-DataStream<String> dataStream1 = env
- .addSource(new MyFlumeSource("localhost", 41414))
- .print();
-~~~
-
-The followings have to be provided for the `MyFlumeSource()` constructor in order:
+The followings have to be provided for the `FlumeSink(…)` constructor in order:
1. The hostname
2. The port number
+3. Serialisation schema
-Similarly to use a Flume connector as a sink in Flink call the `addSink()` function with a new instance of the class which extends `FlumeSink`
+Example:
~~~java
-DataStream<String> dataStream2 = env
- .fromElements("one", "two", "three", "four", "five", "q")
- .addSink(new MyFlumeSink("localhost", 42424));
+stream.addSink(new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
~~~
-The followings have to be provided for the `MyFlumeSink()` constructor in order:
-
-1. The hostname
-2. The port number
-
##### Configuration file<a name="config_file"></a>
An example of a configuration file:
@@ -890,107 +752,38 @@ This connector provides access to datastreams from [RabbitMQ](http://www.rabbitm
Follow the instructions from the [RabbitMQ download page](http://www.rabbitmq.com/download.html). After the installation the server automatically starts and the application connecting to RabbitMQ can be launched.
#### RabbitMQ Source
-An abstract class providing an interface for receiving data from RabbitMQ. By implementing the user must:
-
-* Write a constructor calling the constructor of the abstract class,
-* Write a deserializer function which processes the data coming from RabbitMQ,
-* Stop the source manually when necessary with one of the close functions.
-The implemented class must extend `RabbitMQSource` for example: `RabbitMQSource<String>`
+A class providing an interface for receiving data from RabbitMQ.
-##### Constructor
-An example of an implementation of a constructor:
+The followings have to be provided for the `RMQSource(…)` constructor in order:
-~~~java
-public MyRMQSource(String HOST_NAME, String QUEUE_NAME) {
- super(HOST_NAME, QUEUE_NAME);
-}
-~~~
+1. The hostname
+2. The queue name
+3. Deserialisation schema
-##### Deserializer
-An example of an implemetation of a deserializer:
+Example:
~~~java
-@Override
-public String deserialize(byte[] t) {
- String s = (String) SerializationUtils.deserialize(t);
- String out = s;
- if (s.equals("q")) {
- closeWithoutSend();
- }
- return out;
-}
+DataStream<String> stream = env
+ .addSource(new RMQSource<String>("localhost", "hello", new SimpleStringSchema()))
+ .print();
~~~
-The source closes when it receives the String `"q"`.
-
-##### Close<a name="rmq_source_close"></a>
-Two types of close functions are available, namely `closeWithoutSend()` and `sendAndClose()`. The former closes the connection immediately and no further data will be sent, while the latter closes the connection only when the next message is sent after this call.
-
-Closes the connection only when the next message is sent after this call.
-
-In the example `closeWithoutSend()` is used because here the String `"q"` is meta-message indicating the end of the stream and there is no need to forward it.
-
#### RabbitMQ Sink
-An abstract class providing an interface for sending data to RabbitMQ. By implementing the user must:
-
-* Write a constructor calling the constructor of the abstract class
-* Write a serializer function to send data in the desired form to RabbitMQ
-* Stop the sink manually when necessary with one of the close functions
-
-The implemented class must extend `RabbitMQSink` for example: `RabbitMQSink<String, String>`
-
-##### Constructor
-An example of an implementation of a constructor:
-
-~~~java
-public MyRMQSink(String HOST_NAME, String QUEUE_NAME) {
- super(HOST_NAME, QUEUE_NAME);
-}
-~~~
+A class providing an interface for sending data to RabbitMQ.
-##### Serializer
-An example of an implementation of a serializer.
-
-~~~java
-@Override
-public byte[] serialize(Tuple tuple) {
- if (t.getField(0).equals("q")) {
- sendAndClose();
- }
- return SerializationUtils.serialize(tuple.f0);
-}
-~~~
-
-##### Close
-The API provided is the [same](#rmq_source_close) as the one for `RabbitMQSource`.
-
-#### Building A Topology
-To use a RabbitMQ connector as a source in Flink call the `addSource()` function with a new instance of the class which extends `RabbitMQSource` as parameter:
-
-~~~java
-DataStream<String> dataStream1 = env
- .addSource(new MyRMQSource("localhost", "hello"))
- .print();
-~~~
-
-The followings have to be provided for the `MyRabbitMQSource()` constructor in order:
+The followings have to be provided for the `RMQSink(…)` constructor in order:
1. The hostname
2. The queue name
+3. Serialisation schema
-Similarly to use a RabbitMQ connector as a sink in Flink call the `addSink()` function with a new instance of the class which extends `RabbitMQSink`
+Example:
~~~java
-DataStream<String> dataStream2 = env
- .fromElements("one", "two", "three", "four", "five", "q")
- .addSink(new MyRMQSink("localhost", "hello"));
+stream.addSink(new RMQSink<String>("localhost", "hello", new StringToByteSerializer()));
~~~
-The followings have to be provided for the `MyRabbitMQSink()` constructor in order:
-
-1. The hostname
-1. The queue name
More about RabbitMQ can be found [here](http://www.rabbitmq.com/).
http://git-wip-us.apache.org/repos/asf/flink/blob/acd2d601/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
index cf05627..eb34b80 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -151,7 +151,11 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
/**
* Create a DataStream using a user defined source function for arbitrary
- * source functionality.
+ * source functionality. By default sources have a parallelism of 1.
+ * To enable parallel execution, the user defined source should implement
+ * ParallelSourceFunction or extend RichParallelSourceFunction.
+ * In these cases the resulting source will have the parallelism of the environment.
+ * To change this afterwards call DataStreamSource.setParallelism(int)
*
*/
def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
[22/36] flink git commit: [streaming] Source parallelism + connector
rework
Posted by mb...@apache.org.
[streaming] Source parallelism + connector rework
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e0fd9dfd
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e0fd9dfd
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e0fd9dfd
Branch: refs/heads/release-0.8
Commit: e0fd9dfd0e884251f56965a416e60df0902cd2c7
Parents: 7eaf726
Author: Gyula Fora <gy...@apache.org>
Authored: Tue Dec 23 22:13:22 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:07:50 2015 +0100
----------------------------------------------------------------------
.../streaming/connectors/flume/FlumeSink.java | 64 +++-------
.../streaming/connectors/flume/FlumeSource.java | 61 +++------
.../connectors/flume/FlumeTopology.java | 71 ++---------
.../streaming/connectors/kafka/KafkaSink.java | 49 ++-----
.../streaming/connectors/kafka/KafkaSource.java | 56 +++-----
.../connectors/kafka/KafkaTopology.java | 52 ++------
.../streaming/connectors/rabbitmq/RMQSink.java | 61 +++------
.../connectors/rabbitmq/RMQSource.java | 56 +++-----
.../connectors/rabbitmq/RMQTopology.java | 127 ++++++-------------
.../connectors/twitter/TwitterSource.java | 4 +-
.../connectors/util/DeserializationScheme.java | 42 ++++++
.../streaming/connectors/util/RawScheme.java | 39 ++++++
.../connectors/util/SerializationScheme.java | 33 +++++
.../connectors/util/SimpleStringScheme.java | 40 ++++++
.../streaming/api/datastream/DataStream.java | 2 +-
.../api/datastream/DataStreamSource.java | 77 ++++++-----
.../environment/StreamExecutionEnvironment.java | 84 ++++++++----
.../function/source/GenSequenceFunction.java | 92 ++++++++------
.../function/source/ParallelSourceFunction.java | 26 ++++
.../source/RichParallelSourceFunction.java | 27 ++++
.../api/function/source/RichSourceFunction.java | 50 ++++----
.../apache/flink/streaming/api/SourceTest.java | 19 +--
.../streaming/StreamExecutionEnvironment.scala | 11 +-
23 files changed, 562 insertions(+), 581 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
index a8fc57e..3e68c45 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSink.java
@@ -17,18 +17,19 @@
package org.apache.flink.streaming.connectors.flume;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.util.SerializationScheme;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.FlumeException;
import org.apache.flume.api.RpcClient;
import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.event.EventBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public abstract class FlumeSink<IN> implements SinkFunction<IN> {
+public class FlumeSink<IN> extends RichSinkFunction<IN> {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(FlumeSink.class);
@@ -37,17 +38,17 @@ public abstract class FlumeSink<IN> implements SinkFunction<IN> {
boolean initDone = false;
String host;
int port;
- private boolean sendAndClose = false;
- private boolean closeWithoutSend = false;
+ SerializationScheme<IN, byte[]> scheme;
- public FlumeSink(String host, int port) {
+ public FlumeSink(String host, int port, SerializationScheme<IN, byte[]> scheme) {
this.host = host;
this.port = port;
+ this.scheme = scheme;
}
/**
- * Receives tuples from the Apache Flink {@link DataStream} and forwards them to
- * Apache Flume.
+ * Receives tuples from the Apache Flink {@link DataStream} and forwards
+ * them to Apache Flume.
*
* @param value
* The tuple arriving from the datastream
@@ -60,25 +61,11 @@ public abstract class FlumeSink<IN> implements SinkFunction<IN> {
client.init(host, port);
}
- byte[] data = serialize(value);
- if (!closeWithoutSend) {
- client.sendDataToFlume(data);
- }
- if (sendAndClose) {
- client.close();
- }
+ byte[] data = scheme.serialize(value);
+ client.sendDataToFlume(data);
}
- /**
- * Serializes tuples into byte arrays.
- *
- * @param value
- * The tuple used for the serialization
- * @return The serialized byte array.
- */
- public abstract byte[] serialize(IN value);
-
private class FlinkRpcClientFacade {
private RpcClient client;
private String hostname;
@@ -99,7 +86,8 @@ public abstract class FlumeSink<IN> implements SinkFunction<IN> {
int initCounter = 0;
while (true) {
if (initCounter >= 90) {
- throw new RuntimeException("Cannot establish connection with" + port + " at " + host);
+ throw new RuntimeException("Cannot establish connection with" + port + " at "
+ + host);
}
try {
this.client = RpcClientFactory.getDefaultInstance(hostname, port);
@@ -142,28 +130,10 @@ public abstract class FlumeSink<IN> implements SinkFunction<IN> {
}
}
- /**
- * Closes the RpcClient.
- */
- public void close() {
- client.close();
- }
-
}
- /**
- * Closes the connection only when the next message is sent after this call.
- */
- public void sendAndClose() {
- sendAndClose = true;
- }
-
- /**
- * Closes the connection immediately and no further data will be sent.
- */
- public void closeWithoutSend() {
- client.close();
- closeWithoutSend = true;
+ @Override
+ public void close() {
+ client.client.close();
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
index d5faa34..ceb2be8 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeSource.java
@@ -20,7 +20,8 @@ package org.apache.flink.streaming.connectors.flume;
import java.util.List;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.source.RichSourceFunction;
+import org.apache.flink.streaming.api.function.source.ParallelSourceFunction;
+import org.apache.flink.streaming.connectors.util.DeserializationScheme;
import org.apache.flink.util.Collector;
import org.apache.flume.Context;
import org.apache.flume.channel.ChannelProcessor;
@@ -28,15 +29,18 @@ import org.apache.flume.source.AvroSource;
import org.apache.flume.source.avro.AvroFlumeEvent;
import org.apache.flume.source.avro.Status;
-public abstract class FlumeSource<OUT> extends RichSourceFunction<OUT> {
+public class FlumeSource<OUT> implements ParallelSourceFunction<OUT> {
private static final long serialVersionUID = 1L;
String host;
String port;
+ DeserializationScheme<OUT> scheme;
+ volatile boolean finished = false;
- FlumeSource(String host, int port) {
+ FlumeSource(String host, int port, DeserializationScheme<OUT> scheme) {
this.host = host;
this.port = Integer.toString(port);
+ this.scheme = scheme;
}
public class MyAvroSource extends AvroSource {
@@ -48,7 +52,8 @@ public abstract class FlumeSource<OUT> extends RichSourceFunction<OUT> {
*
* @param avroEvent
* The event that should be sent to the dataStream
- * @return A {@link Status}.OK message if sending the event was successful.
+ * @return A {@link Status}.OK message if sending the event was
+ * successful.
*/
@Override
public Status append(AvroFlumeEvent avroEvent) {
@@ -82,30 +87,21 @@ public abstract class FlumeSource<OUT> extends RichSourceFunction<OUT> {
*/
private void collect(AvroFlumeEvent avroEvent) {
byte[] b = avroEvent.getBody().array();
- OUT tuple = FlumeSource.this.deserialize(b);
- if (!closeWithoutSend) {
- collector.collect(tuple);
- }
- if (sendAndClose) {
- sendDone = true;
+ OUT out = FlumeSource.this.scheme.deserialize(b);
+
+ if (scheme.isEndOfStream(out)) {
+ FlumeSource.this.finished = true;
+ this.stop();
+ FlumeSource.this.notifyAll();
+ } else {
+ collector.collect(out);
}
+
}
}
MyAvroSource avroSource;
- private volatile boolean closeWithoutSend = false;
- private boolean sendAndClose = false;
- private volatile boolean sendDone = false;
-
- /**
- * Deserializes the incoming data.
- *
- * @param message
- * The incoming message in a byte array
- * @return The deserialized message in the required format.
- */
- public abstract OUT deserialize(byte[] message);
/**
* Configures the AvroSource. Also sets the collector so the application can
@@ -138,26 +134,9 @@ public abstract class FlumeSource<OUT> extends RichSourceFunction<OUT> {
public void invoke(Collector<OUT> collector) throws Exception {
configureAvroSource(collector);
avroSource.start();
- while (true) {
- if (closeWithoutSend || sendDone) {
- break;
- }
+ while (!finished) {
+ this.wait();
}
- avroSource.stop();
- }
-
- /**
- * Closes the connection only when the next message is sent after this call.
- */
- public void sendAndClose() {
- sendAndClose = true;
- }
-
- /**
- * Closes the connection immediately and no further data will be sent.
- */
- public void closeWithoutSend() {
- closeWithoutSend = true;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
index 8dc3c5a..366b1a5 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/flume/FlumeTopology.java
@@ -17,80 +17,33 @@
package org.apache.flink.streaming.connectors.flume;
-import org.apache.commons.lang.SerializationUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.connectors.util.SerializationScheme;
+import org.apache.flink.streaming.connectors.util.SimpleStringScheme;
public class FlumeTopology {
- private static final Logger LOG = LoggerFactory.getLogger(FlumeTopology.class);
- public static class MyFlumeSink extends FlumeSink<String> {
- private static final long serialVersionUID = 1L;
+ public static void main(String[] args) throws Exception {
- public MyFlumeSink(String host, int port) {
- super(host, port);
- }
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
- @Override
- public byte[] serialize(String tuple) {
- if (tuple.equals("q")) {
- try {
- sendAndClose();
- } catch (Exception e) {
- throw new RuntimeException("Error while closing Flume connection with " + port
- + " at " + host, e);
- }
- }
- return SerializationUtils.serialize(tuple);
- }
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream1 = env.addSource(
+ new FlumeSource<String>("localhost", 41414, new SimpleStringScheme())).addSink(
+ new FlumeSink<String>("localhost", 42424, new StringToByteSerializer()));
+ env.execute();
}
- public static final class MyFlumePrintSink implements SinkFunction<String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(String value) {
- if (LOG.isInfoEnabled()) {
- LOG.info("String: <{}> arrived from Flume", value);
- }
- }
-
- }
+ public static class StringToByteSerializer implements SerializationScheme<String, byte[]> {
- public static class MyFlumeSource extends FlumeSource<String> {
private static final long serialVersionUID = 1L;
- MyFlumeSource(String host, int port) {
- super(host, port);
- }
-
@Override
- public String deserialize(byte[] msg) {
- String s = (String) SerializationUtils.deserialize(msg);
- if (s.equals("q")) {
- closeWithoutSend();
- }
- return s;
+ public byte[] serialize(String element) {
+ return element.getBytes();
}
-
}
- public static void main(String[] args) throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream1 = env.addSource(new MyFlumeSource("localhost", 41414))
- .addSink(new MyFlumePrintSink());
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream2 = env.fromElements("one", "two", "three", "four", "five",
- "q").addSink(new MyFlumeSink("localhost", 42424));
-
- env.execute();
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
index f7f998f..71e25f2 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSink.java
@@ -23,22 +23,24 @@ import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.util.SerializationScheme;
-public abstract class KafkaSink<IN, OUT> implements SinkFunction<IN> {
+public class KafkaSink<IN, OUT> extends RichSinkFunction<IN> {
private static final long serialVersionUID = 1L;
private kafka.javaapi.producer.Producer<Integer, OUT> producer;
private Properties props;
private String topicId;
private String brokerAddr;
- private boolean sendAndClose = false;
- private boolean closeWithoutSend = false;
private boolean initDone = false;
+ private SerializationScheme<IN, OUT> scheme;
- public KafkaSink(String topicId, String brokerAddr) {
+ public KafkaSink(String topicId, String brokerAddr,
+ SerializationScheme<IN, OUT> serializationScheme) {
this.topicId = topicId;
this.brokerAddr = brokerAddr;
+ this.scheme = serializationScheme;
}
@@ -60,49 +62,22 @@ public abstract class KafkaSink<IN, OUT> implements SinkFunction<IN> {
/**
* Called when new data arrives to the sink, and forwards it to Kafka.
*
- * @param value
+ * @param next
* The incoming data
*/
@Override
- public void invoke(IN value) {
+ public void invoke(IN next) {
if (!initDone) {
initialize();
}
- OUT out = serialize(value);
- KeyedMessage<Integer, OUT> data = new KeyedMessage<Integer, OUT>(topicId, out);
+ producer.send(new KeyedMessage<Integer, OUT>(topicId, scheme.serialize(next)));
- if (!closeWithoutSend) {
- producer.send(data);
- }
-
- if (sendAndClose) {
- producer.close();
- }
}
- /**
- * Serializes tuples into byte arrays.
- *
- * @param value
- * The tuple used for the serialization
- * @return The serialized byte array.
- */
- public abstract OUT serialize(IN value);
-
- /**
- * Closes the connection immediately and no further data will be sent.
- */
- public void closeWithoutSend() {
+ @Override
+ public void close() {
producer.close();
- closeWithoutSend = true;
- }
-
- /**
- * Closes the connection only when the next message is sent after this call.
- */
- public void sendAndClose() {
- sendAndClose = true;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
index 75c3983..49e9144 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSource.java
@@ -17,7 +17,7 @@
package org.apache.flink.streaming.connectors.kafka;
-import java.util.HashMap;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -27,28 +27,29 @@ import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.source.RichSourceFunction;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.connectors.util.DeserializationScheme;
import org.apache.flink.util.Collector;
-public abstract class KafkaSource<OUT> extends RichSourceFunction<OUT> {
+public class KafkaSource<OUT> extends RichParallelSourceFunction<OUT> {
private static final long serialVersionUID = 1L;
private final String zkQuorum;
private final String groupId;
private final String topicId;
- private final int numThreads;
private ConsumerConnector consumer;
- private boolean closeWithoutSend = false;
- private boolean sendAndClose = false;
+ private DeserializationScheme<OUT> scheme;
OUT outTuple;
- public KafkaSource(String zkQuorum, String groupId, String topicId, int numThreads) {
+ public KafkaSource(String zkQuorum, String groupId, String topicId,
+ DeserializationScheme<OUT> deserializationScheme) {
this.zkQuorum = zkQuorum;
this.groupId = groupId;
this.topicId = topicId;
- this.numThreads = numThreads;
+ this.scheme = deserializationScheme;
}
/**
@@ -58,7 +59,7 @@ public abstract class KafkaSource<OUT> extends RichSourceFunction<OUT> {
Properties props = new Properties();
props.put("zookeeper.connect", zkQuorum);
props.put("group.id", groupId);
- props.put("zookeeper.session.timeout.ms", "400");
+ props.put("zookeeper.session.timeout.ms", "2000");
props.put("zookeeper.sync.time.ms", "200");
props.put("auto.commit.interval.ms", "1000");
consumer = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(props));
@@ -72,48 +73,25 @@ public abstract class KafkaSource<OUT> extends RichSourceFunction<OUT> {
*/
@Override
public void invoke(Collector<OUT> collector) throws Exception {
- initializeConnection();
- Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
- topicCountMap.put(topicId, numThreads);
Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer
- .createMessageStreams(topicCountMap);
+ .createMessageStreams(Collections.singletonMap(topicId, 1));
+
KafkaStream<byte[], byte[]> stream = consumerMap.get(topicId).get(0);
ConsumerIterator<byte[], byte[]> it = stream.iterator();
while (it.hasNext()) {
- OUT out = deserialize(it.next().message());
- if (closeWithoutSend) {
+ OUT out = scheme.deserialize(it.next().message());
+ if (scheme.isEndOfStream(out)) {
break;
}
collector.collect(out);
- if (sendAndClose) {
- break;
- }
}
consumer.shutdown();
}
- /**
- * Deserializes the incoming data.
- *
- * @param message
- * The incoming message in a byte array
- * @return The deserialized message in the required format.
- */
- public abstract OUT deserialize(byte[] message);
-
- /**
- * Closes the connection immediately and no further data will be sent.
- */
- public void closeWithoutSend() {
- closeWithoutSend = true;
- }
-
- /**
- * Closes the connection only when the next message is sent after this call.
- */
- public void sendAndClose() {
- sendAndClose = true;
+ @Override
+ public void open(Configuration config) {
+ initializeConnection();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
index 4d043c4..16f123e 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaTopology.java
@@ -21,13 +21,14 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.sink.SinkFunction;
import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.connectors.util.SimpleStringScheme;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class KafkaTopology {
private static final Logger LOG = LoggerFactory.getLogger(KafkaTopology.class);
-
+
public static final class MySource implements SourceFunction<String> {
private static final long serialVersionUID = 1L;
@@ -41,41 +42,6 @@ public class KafkaTopology {
}
}
- public static final class MyKafkaSource extends KafkaSource<String> {
- private static final long serialVersionUID = 1L;
-
- public MyKafkaSource(String zkQuorum, String groupId, String topicId, int numThreads) {
- super(zkQuorum, groupId, topicId, numThreads);
- }
-
- @Override
- public String deserialize(byte[] msg) {
- String s = new String(msg);
- if (s.equals("q")) {
- closeWithoutSend();
- }
- return new String(s);
- }
-
- }
-
- public static final class MyKafkaSink extends KafkaSink<String, String> {
- private static final long serialVersionUID = 1L;
-
- public MyKafkaSink(String topicId, String brokerAddr) {
- super(topicId, brokerAddr);
- }
-
- @Override
- public String serialize(String tuple) {
- if (tuple.equals("q")) {
- sendAndClose();
- }
- return tuple;
- }
-
- }
-
public static final class MyKafkaPrintSink implements SinkFunction<String> {
private static final long serialVersionUID = 1L;
@@ -87,21 +53,19 @@ public class KafkaTopology {
}
}
- private static final int SOURCE_PARALELISM = 1;
-
public static void main(String[] args) throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
@SuppressWarnings("unused")
DataStream<String> stream1 = env
- .addSource(new MyKafkaSource("localhost:2181", "group", "test", 1)).setParallelism(1)
- .addSink(new MyKafkaPrintSink());
+ .addSource(
+ new KafkaSource<String>("localhost:2181", "group", "test",
+ new SimpleStringScheme())).addSink(new MyKafkaPrintSink());
@SuppressWarnings("unused")
- DataStream<String> stream2 = env
- .addSource(new MySource())
- .addSink(new MyKafkaSink("test", "localhost:9092"));
+ DataStream<String> stream2 = env.addSource(new MySource()).addSink(
+ new KafkaSink<String, String>("test", "localhost:9092", new SimpleStringScheme()));
env.execute();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
index f7197d5..c4f4615 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSink.java
@@ -19,32 +19,32 @@ package org.apache.flink.streaming.connectors.rabbitmq;
import java.io.IOException;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.function.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.util.SerializationScheme;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
-public abstract class RMQSink<IN> implements SinkFunction<IN> {
+public class RMQSink<IN> extends RichSinkFunction<IN> {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(RMQSink.class);
- private boolean sendAndClose = false;
- private boolean closeWithoutSend = false;
-
private String QUEUE_NAME;
private String HOST_NAME;
private transient ConnectionFactory factory;
private transient Connection connection;
private transient Channel channel;
- private boolean initDone = false;
+ private SerializationScheme<IN, byte[]> scheme;
- public RMQSink(String HOST_NAME, String QUEUE_NAME) {
+ public RMQSink(String HOST_NAME, String QUEUE_NAME, SerializationScheme<IN, byte[]> scheme) {
this.HOST_NAME = HOST_NAME;
this.QUEUE_NAME = QUEUE_NAME;
+ this.scheme = scheme;
}
/**
@@ -60,8 +60,6 @@ public abstract class RMQSink<IN> implements SinkFunction<IN> {
} catch (IOException e) {
throw new RuntimeException(e);
}
-
- initDone = true;
}
/**
@@ -72,37 +70,21 @@ public abstract class RMQSink<IN> implements SinkFunction<IN> {
*/
@Override
public void invoke(IN value) {
- if (!initDone) {
- initializeConnection();
- }
-
try {
channel.queueDeclare(QUEUE_NAME, false, false, false, null);
- byte[] msg = serialize(value);
- if (!closeWithoutSend) {
- channel.basicPublish("", QUEUE_NAME, null, msg);
- }
+ byte[] msg = scheme.serialize(value);
+
+ channel.basicPublish("", QUEUE_NAME, null, msg);
+
} catch (IOException e) {
if (LOG.isErrorEnabled()) {
LOG.error("Cannot send RMQ message {} at {}", QUEUE_NAME, HOST_NAME);
}
}
- if (sendAndClose) {
- closeChannel();
- }
}
/**
- * Serializes tuples into byte arrays.
- *
- * @param value
- * The tuple used for the serialization
- * @return The serialized byte array.
- */
- public abstract byte[] serialize(IN value);
-
- /**
* Closes the connection.
*/
private void closeChannel() {
@@ -110,25 +92,20 @@ public abstract class RMQSink<IN> implements SinkFunction<IN> {
channel.close();
connection.close();
} catch (IOException e) {
- throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME + " at "
- + HOST_NAME, e);
+ throw new RuntimeException("Error while closing RMQ connection with " + QUEUE_NAME
+ + " at " + HOST_NAME, e);
}
}
- /**
- * Closes the connection immediately and no further data will be sent.
- */
- public void closeWithoutSend() {
- closeChannel();
- closeWithoutSend = true;
+ @Override
+ public void open(Configuration config) {
+ initializeConnection();
}
- /**
- * Closes the connection only when the next message is sent after this call.
- */
- public void sendAndClose() {
- sendAndClose = true;
+ @Override
+ public void close() {
+ closeChannel();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
index ee83ffd..240deab 100755
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQSource.java
@@ -19,8 +19,10 @@ package org.apache.flink.streaming.connectors.rabbitmq;
import java.io.IOException;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.function.source.RichSourceFunction;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.connectors.util.DeserializationScheme;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,15 +32,13 @@ import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.QueueingConsumer;
-public abstract class RMQSource<OUT> extends RichSourceFunction<OUT> {
+public class RMQSource<OUT> extends RichParallelSourceFunction<OUT> {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(RMQSource.class);
private final String QUEUE_NAME;
private final String HOST_NAME;
- private boolean closeWithoutSend = false;
- private boolean sendAndClose = false;
private transient ConnectionFactory factory;
private transient Connection connection;
@@ -46,9 +46,11 @@ public abstract class RMQSource<OUT> extends RichSourceFunction<OUT> {
private transient QueueingConsumer consumer;
private transient QueueingConsumer.Delivery delivery;
- OUT outTuple;
+ private DeserializationScheme<OUT> scheme;
- public RMQSource(String HOST_NAME, String QUEUE_NAME) {
+ OUT out;
+
+ public RMQSource(String HOST_NAME, String QUEUE_NAME, DeserializationScheme<OUT> scheme) {
this.HOST_NAME = HOST_NAME;
this.QUEUE_NAME = QUEUE_NAME;
}
@@ -79,7 +81,6 @@ public abstract class RMQSource<OUT> extends RichSourceFunction<OUT> {
*/
@Override
public void invoke(Collector<OUT> collector) throws Exception {
- initializeConnection();
while (true) {
@@ -91,17 +92,23 @@ public abstract class RMQSource<OUT> extends RichSourceFunction<OUT> {
}
}
- outTuple = deserialize(delivery.getBody());
- if (closeWithoutSend) {
+ out = scheme.deserialize(delivery.getBody());
+ if (scheme.isEndOfStream(out)) {
break;
} else {
- collector.collect(outTuple);
- }
- if (sendAndClose) {
- break;
+ collector.collect(out);
}
}
+ }
+
+ @Override
+ public void open(Configuration config) {
+ initializeConnection();
+ }
+
+ @Override
+ public void close() {
try {
connection.close();
} catch (IOException e) {
@@ -111,27 +118,4 @@ public abstract class RMQSource<OUT> extends RichSourceFunction<OUT> {
}
- /**
- * Deserializes the incoming data.
- *
- * @param message
- * The incoming message in a byte array
- * @return The deserialized message in the required format.
- */
- public abstract OUT deserialize(byte[] message);
-
- /**
- * Closes the connection immediately and no further data will be sent.
- */
- public void closeWithoutSend() {
- closeWithoutSend = true;
- }
-
- /**
- * Closes the connection only when the next message is sent after this call.
- */
- public void sendAndClose() {
- sendAndClose = true;
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
index f640ee0..4f82b24 100755
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/rabbitmq/RMQTopology.java
@@ -1,95 +1,52 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.rabbitmq;
-
-import org.apache.commons.lang.SerializationUtils;
+ */
+
+package org.apache.flink.streaming.connectors.rabbitmq;
+
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.connectors.kafka.KafkaTopology;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class RMQTopology {
- private static final Logger LOG = LoggerFactory.getLogger(KafkaTopology.class);
-
- public static final class MyRMQSink extends RMQSink<String> {
- public MyRMQSink(String HOST_NAME, String QUEUE_NAME) {
- super(HOST_NAME, QUEUE_NAME);
- }
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public byte[] serialize(String t) {
- if (t.equals("q")) {
- sendAndClose();
- }
- return SerializationUtils.serialize((String) t);
- }
-
- }
+import org.apache.flink.streaming.connectors.util.SerializationScheme;
+import org.apache.flink.streaming.connectors.util.SimpleStringScheme;
+
+public class RMQTopology {
+
+ public static void main(String[] args) throws Exception {
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream1 = env.addSource(
+ new RMQSource<String>("localhost", "hello", new SimpleStringScheme())).print();
+
+ @SuppressWarnings("unused")
+ DataStream<String> dataStream2 = env.fromElements("one", "two", "three", "four", "five",
+ "q").addSink(
+ new RMQSink<String>("localhost", "hello", new StringToByteSerializer()));
+
+ env.execute();
+ }
+
+ public static class StringToByteSerializer implements SerializationScheme<String, byte[]> {
- public static final class MyRMQPrintSink implements SinkFunction<String> {
private static final long serialVersionUID = 1L;
@Override
- public void invoke(String value) {
- if (LOG.isInfoEnabled()) {
- LOG.info("String: <{}> arrived from RMQ", value);
- }
+ public byte[] serialize(String element) {
+ return element.getBytes();
}
-
}
-
- public static final class MyRMQSource extends RMQSource<String> {
-
- public MyRMQSource(String HOST_NAME, String QUEUE_NAME) {
- super(HOST_NAME, QUEUE_NAME);
- }
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String deserialize(byte[] t) {
- String s = (String) SerializationUtils.deserialize(t);
- if (s.equals("q")) {
- closeWithoutSend();
- }
- return s;
- }
-
- }
-
- public static void main(String[] args) throws Exception {
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream1 = env
- .addSource(new MyRMQSource("localhost", "hello"))
- .addSink(new MyRMQPrintSink());
-
- @SuppressWarnings("unused")
- DataStream<String> dataStream2 = env
- .fromElements("one", "two", "three", "four", "five", "q")
- .addSink(new MyRMQSink("localhost", "hello"));
-
- env.execute();
- }
-}
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
index 3ffa8ae..ddb2538 100644
--- a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/twitter/TwitterSource.java
@@ -25,7 +25,7 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.function.source.RichSourceFunction;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.function.source.SourceFunction;
import org.apache.flink.util.Collector;
import org.slf4j.Logger;
@@ -43,7 +43,7 @@ import com.twitter.hbc.httpclient.auth.OAuth1;
* Implementation of {@link SourceFunction} specialized to emit tweets from
* Twitter. It can connect to Twitter Streaming API, collect tweets and
*/
-public class TwitterSource extends RichSourceFunction<String> {
+public class TwitterSource extends RichParallelSourceFunction<String> {
private static final Logger LOG = LoggerFactory.getLogger(TwitterSource.class);
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java
new file mode 100644
index 0000000..43420a3
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/DeserializationScheme.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+import java.io.Serializable;
+
+public interface DeserializationScheme<T> extends Serializable {
+
+ /**
+ * Deserializes the incoming data.
+ *
+ * @param message
+ * The incoming message in a byte array
+ * @return The deserialized message in the required format.
+ */
+ public T deserialize(byte[] message);
+
+ /**
+ * Method to decide whether the element signals the end of the stream. If
+ * true is returned the element won't be emitted
+ *
+ * @param nextElement
+ * The element to test for end signal
+ * @return The end signal, if true the stream shuts down
+ */
+ public boolean isEndOfStream(T nextElement);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java
new file mode 100644
index 0000000..34db00e
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/RawScheme.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+public class RawScheme implements DeserializationScheme<byte[]>,
+ SerializationScheme<byte[], byte[]> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public byte[] deserialize(byte[] message) {
+ return message;
+ }
+
+ @Override
+ public boolean isEndOfStream(byte[] nextElement) {
+ return false;
+ }
+
+ @Override
+ public byte[] serialize(byte[] element) {
+ return element;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java
new file mode 100644
index 0000000..bb25885
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SerializationScheme.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+import java.io.Serializable;
+
+public interface SerializationScheme<T,R> extends Serializable {
+
+ /**
+ * Serializes the incoming element to a specified type.
+ *
+ * @param element
+ * The incoming element to be serialized
+ * @return The serialized element.
+ */
+ public R serialize(T element);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java
new file mode 100644
index 0000000..9d1ce6c
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/util/SimpleStringScheme.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.util;
+
+public class SimpleStringScheme implements DeserializationScheme<String>,
+ SerializationScheme<String, String> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String deserialize(byte[] message) {
+ return new String(message);
+ }
+
+ @Override
+ public boolean isEndOfStream(String nextElement) {
+ return false;
+ }
+
+ @Override
+ public String serialize(String element) {
+ return element;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 422df5b..8d0020e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -1005,7 +1005,7 @@ public class DataStream<OUT> {
protected <R> DataStream<OUT> addIterationSource(Integer iterationID, long waitTime) {
- DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null);
+ DataStream<R> returnStream = new DataStreamSource<R>(environment, "iterationSource", null, true);
jobGraphBuilder.addIterationHead(returnStream.getId(), this.getId(), iterationID,
degreeOfParallelism, waitTime);
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
index 978ea42..a649f59 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSource.java
@@ -1,38 +1,47 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
-/**
- * The DataStreamSource represents the starting point of a DataStream.
- *
- * @param <OUT>
- * Type of the DataStream created.
- */
-public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataStreamSource<OUT>> {
-
- public DataStreamSource(StreamExecutionEnvironment environment, String operatorType, TypeInformation<OUT> outTypeInfo) {
- super(environment, operatorType, outTypeInfo);
- }
-
- public DataStreamSource(DataStream<OUT> dataStream) {
- super(dataStream);
- }
-}
+
+/**
+ * The DataStreamSource represents the starting point of a DataStream.
+ *
+ * @param <OUT>
+ * Type of the DataStream created.
+ */
+public class DataStreamSource<OUT> extends SingleOutputStreamOperator<OUT, DataStreamSource<OUT>> {
+
+ boolean isParallel;
+
+ public DataStreamSource(StreamExecutionEnvironment environment, String operatorType,
+ TypeInformation<OUT> outTypeInfo, boolean isParallel) {
+ super(environment, operatorType, outTypeInfo);
+ this.isParallel = isParallel;
+ }
+
+ @Override
+ public DataStreamSource<OUT> setParallelism(int dop) {
+ if (dop > 1 && !isParallel) {
+ throw new IllegalArgumentException("Source: " + this.id + " is not a parallel source");
+ } else {
+ return (DataStreamSource<OUT>) super.setParallelism(dop);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 5bf2164..c7792a4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -40,6 +40,8 @@ import org.apache.flink.streaming.api.function.source.FileSourceFunction;
import org.apache.flink.streaming.api.function.source.FileStreamFunction;
import org.apache.flink.streaming.api.function.source.FromElementsFunction;
import org.apache.flink.streaming.api.function.source.GenSequenceFunction;
+import org.apache.flink.streaming.api.function.source.ParallelSourceFunction;
+import org.apache.flink.streaming.api.function.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
import org.apache.flink.streaming.api.function.source.SourceFunction;
import org.apache.flink.streaming.api.invokable.SourceInvokable;
@@ -222,7 +224,7 @@ public abstract class StreamExecutionEnvironment {
*/
public DataStreamSource<String> readTextStream(String filePath) {
checkIfFileExists(filePath);
- return addSource(new FileStreamFunction(filePath));
+ return addSource(new FileStreamFunction(filePath), null, "textStream");
}
private static void checkIfFileExists(String filePath) {
@@ -260,14 +262,10 @@ public abstract class StreamExecutionEnvironment {
}
TypeInformation<OUT> outTypeInfo = TypeExtractor.getForObject(data[0]);
- DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "elements",
- outTypeInfo);
SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
- jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
- null, outTypeInfo, "source", 1);
- return returnStream;
+ return addSource(function, outTypeInfo, "elements");
}
/**
@@ -292,13 +290,9 @@ public abstract class StreamExecutionEnvironment {
}
TypeInformation<OUT> outTypeInfo = TypeExtractor.getForObject(data.iterator().next());
- DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "collection",
- outTypeInfo);
-
- jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(
- new FromElementsFunction<OUT>(data)), null, outTypeInfo, "source", 1);
+ SourceFunction<OUT> function = new FromElementsFunction<OUT>(data);
- return returnStream;
+ return addSource(function, outTypeInfo, "collection");
}
/**
@@ -316,7 +310,8 @@ public abstract class StreamExecutionEnvironment {
* @return A DataStream, containing the strings received from socket.
*/
public DataStreamSource<String> socketTextStream(String hostname, int port, char delimiter) {
- return addSource(new SocketTextStreamFunction(hostname, port, delimiter));
+ return addSource(new SocketTextStreamFunction(hostname, port, delimiter), null,
+ "socketStrean");
}
/**
@@ -348,20 +343,26 @@ public abstract class StreamExecutionEnvironment {
if (from > to) {
throw new IllegalArgumentException("Start of sequence must not be greater than the end");
}
- return addSource(new GenSequenceFunction(from, to));
+ return addSource(new GenSequenceFunction(from, to), null, "sequence");
}
private DataStreamSource<String> addFileSource(InputFormat<String, ?> inputFormat,
TypeInformation<String> typeInfo) {
FileSourceFunction function = new FileSourceFunction(inputFormat, typeInfo);
- DataStreamSource<String> returnStream = addSource(function);
+ DataStreamSource<String> returnStream = addSource(function, null, "fileSource");
jobGraphBuilder.setInputFormat(returnStream.getId(), inputFormat);
return returnStream;
}
/**
* Create a DataStream using a user defined source function for arbitrary
- * source functionality.
+ * source functionality.</p> By default sources have a parallelism of 1. To
+ * enable parallel execution, the user defined source should implement
+ * {@link ParallelSourceFunction} or extend
+ * {@link RichParallelSourceFunction}. In these cases the resulting source
+ * will have the parallelism of the environment. To change this afterwards
+ * call {@link DataStreamSource#setParallelism(int)}
+ *
*
* @param function
* the user defined function
@@ -370,22 +371,19 @@ public abstract class StreamExecutionEnvironment {
* @return the data stream constructed
*/
public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function) {
- TypeInformation<OUT> outTypeInfo = TypeExtractor.createTypeInfo(SourceFunction.class,
- function.getClass(), 0, null, null);
-
- DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "source", outTypeInfo);
-
- jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
- null, outTypeInfo, "source", 1);
-
- return returnStream;
+ return addSource(function, null);
}
/**
* Ads a data source with a custom type information thus opening a
* {@link DataStream}. Only in very special cases does the user need to
* support type information. Otherwise use
- * {@link #addSource(SourceFunction)}
+ * {@link #addSource(SourceFunction)} </p> By default sources have a
+ * parallelism of 1. To enable parallel execution, the user defined source
+ * should implement {@link ParallelSourceFunction} or extend
+ * {@link RichParallelSourceFunction}. In these cases the resulting source
+ * will have the parallelism of the environment. To change this afterwards
+ * call {@link DataStreamSource#setParallelism(int)}
*
* @param function
* the user defined function
@@ -397,11 +395,41 @@ public abstract class StreamExecutionEnvironment {
*/
public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
TypeInformation<OUT> outTypeInfo) {
+ return addSource(function, outTypeInfo, function.getClass().getName());
+ }
+
+ /**
+ * Ads a data source with a custom type information thus opening a
+ * {@link DataStream}. Only in very special cases does the user need to
+ * support type information. Otherwise use
+ * {@link #addSource(SourceFunction)}
+ *
+ * @param function
+ * the user defined function
+ * @param outTypeInfo
+ * the user defined type information for the stream
+ * @param sourceName
+ * Name of the data source
+ * @param <OUT>
+ * type of the returned stream
+ * @return the data stream constructed
+ */
+ private <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
+ TypeInformation<OUT> outTypeInfo, String sourceName) {
+
+ if (outTypeInfo == null) {
+ outTypeInfo = TypeExtractor.createTypeInfo(SourceFunction.class, function.getClass(),
+ 0, null, null);
+ }
+
+ boolean isParallel = function instanceof ParallelSourceFunction;
+ int dop = isParallel ? getDegreeOfParallelism() : 1;
- DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "source", outTypeInfo);
+ DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, sourceName,
+ outTypeInfo, isParallel);
jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
- null, outTypeInfo, "source", 1);
+ null, outTypeInfo, sourceName, dop);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
index 69601ff..3afd06e 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/GenSequenceFunction.java
@@ -1,45 +1,53 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.Collector;
-
-/**
- * Source Function used to generate the number sequence
- *
- */
-public class GenSequenceFunction implements SourceFunction<Long> {
-
- private static final long serialVersionUID = 1L;
-
- long from;
- long to;
-
- public GenSequenceFunction(long from, long to) {
- this.from = from;
- this.to = to;
- }
-
- @Override
- public void invoke(Collector<Long> collector) throws Exception {
- for (long i = from; i <= to; i++) {
- collector.collect(i);
- }
- }
-
-}
+import org.apache.flink.util.NumberSequenceIterator;
+
+/**
+ * Source Function used to generate the number sequence
+ *
+ */
+public class GenSequenceFunction extends RichParallelSourceFunction<Long> {
+
+ private static final long serialVersionUID = 1L;
+
+ private NumberSequenceIterator fullIterator;
+ private NumberSequenceIterator splitIterator;
+
+ public GenSequenceFunction(long from, long to) {
+ fullIterator = new NumberSequenceIterator(from, to);
+ }
+
+ @Override
+ public void invoke(Collector<Long> collector) throws Exception {
+ while (splitIterator.hasNext()) {
+ collector.collect(splitIterator.next());
+ }
+ }
+
+ @Override
+ public void open(Configuration config) {
+ int splitNumber = getRuntimeContext().getIndexOfThisSubtask();
+ int numOfSubTasks = getRuntimeContext().getNumberOfParallelSubtasks();
+ splitIterator = fullIterator.split(numOfSubTasks)[splitNumber];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
new file mode 100644
index 0000000..46d4fe9
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.util.Collector;
+
+public interface ParallelSourceFunction<OUT> extends SourceFunction<OUT> {
+
+ public void invoke(Collector<OUT> collector) throws Exception;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
new file mode 100644
index 0000000..5bbfd4c
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichParallelSourceFunction.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+
+public abstract class RichParallelSourceFunction<OUT> extends AbstractRichFunction implements
+ ParallelSourceFunction<OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
index d212c3c..4b947c7 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/RichSourceFunction.java
@@ -1,27 +1,27 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
* limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.api.common.functions.AbstractRichFunction;
-
-public abstract class RichSourceFunction<OUT> extends AbstractRichFunction implements
- SourceFunction<OUT> {
-
- private static final long serialVersionUID = 1L;
-
-}
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+
+public abstract class RichSourceFunction<OUT> extends AbstractRichFunction implements
+ SourceFunction<OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
index 156a8f2..9be7de6 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/SourceTest.java
@@ -28,7 +28,6 @@ import java.util.Arrays;
import java.util.List;
import org.apache.flink.streaming.api.function.source.FromElementsFunction;
-import org.apache.flink.streaming.api.function.source.GenSequenceFunction;
import org.apache.flink.streaming.api.function.source.SocketTextStreamFunction;
import org.apache.flink.streaming.util.MockCollector;
import org.apache.flink.streaming.util.MockSource;
@@ -39,21 +38,16 @@ public class SourceTest {
@Test
public void fromElementsTest() {
List<Integer> expectedList = Arrays.asList(1, 2, 3);
- List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(1, 2, 3));
+ List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(1,
+ 2, 3));
assertEquals(expectedList, actualList);
}
@Test
public void fromCollectionTest() {
List<Integer> expectedList = Arrays.asList(1, 2, 3);
- List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(Arrays.asList(1, 2, 3)));
- assertEquals(expectedList, actualList);
- }
-
- @Test
- public void genSequenceTest() {
- List<Long> expectedList = Arrays.asList(1L, 2L, 3L);
- List<Long> actualList = MockSource.createAndExecute(new GenSequenceFunction(1, 3));
+ List<Integer> actualList = MockSource.createAndExecute(new FromElementsFunction<Integer>(
+ Arrays.asList(1, 2, 3)));
assertEquals(expectedList, actualList);
}
@@ -62,14 +56,15 @@ public class SourceTest {
List<String> expectedList = Arrays.asList("a", "b", "c");
List<String> actualList = new ArrayList<String>();
- byte[] data = {'a', '\n', 'b', '\n', 'c', '\n'};
+ byte[] data = { 'a', '\n', 'b', '\n', 'c', '\n' };
Socket socket = mock(Socket.class);
when(socket.getInputStream()).thenReturn(new ByteArrayInputStream(data));
when(socket.isClosed()).thenReturn(false);
when(socket.isConnected()).thenReturn(true);
- new SocketTextStreamFunction("", 0, '\n').streamFromSocket(new MockCollector<String>(actualList), socket);
+ new SocketTextStreamFunction("", 0, '\n').streamFromSocket(new MockCollector<String>(
+ actualList), socket);
assertEquals(expectedList, actualList);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/e0fd9dfd/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index 9c66b24..bcd586e 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -145,14 +145,11 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
data: Seq[T]): DataStream[T] = {
Validate.notNull(data, "Data must not be null.")
val typeInfo = implicitly[TypeInformation[T]]
- val returnStream = new DataStreamSource[T](javaEnv,
- "elements", typeInfo);
- javaEnv.getJobGraphBuilder.addStreamVertex(returnStream.getId(),
- new SourceInvokable[T](new FromElementsFunction[T](scala.collection.JavaConversions
- .asJavaCollection(data))), null, typeInfo,
- "source", 1);
- returnStream
+ val sourceFunction = new FromElementsFunction[T](scala.collection.JavaConversions
+ .asJavaCollection(data))
+
+ javaEnv.addSource(sourceFunction, typeInfo)
}
/**
[03/36] flink git commit: [scala] [streaming] Extended scala data
stream functionality to include simple operators
Posted by mb...@apache.org.
[scala] [streaming] Extended scala data stream functionality to include simple operators
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/40efecf7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/40efecf7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/40efecf7
Branch: refs/heads/release-0.8
Commit: 40efecf7cf79deb02f070bedc9980c91886b66ef
Parents: 6b1fd15
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Dec 12 00:12:49 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:49:30 2015 +0100
----------------------------------------------------------------------
.../streaming/api/datastream/DataStream.java | 28 +++
.../api/datastream/GroupedDataStream.java | 4 +
.../flink/api/scala/streaming/DataStream.scala | 233 ++++++++++++++++++-
3 files changed, 257 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/40efecf7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 04929c1..1cf8d72 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -677,6 +677,20 @@ public class DataStream<OUT> {
public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
return this.minBy(positionToMinBy, true);
}
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * minimum value at the given position, if more elements have the minimum
+ * value at the given position, the operator returns the first one by
+ * default.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(String positionToMinBy) {
+ return this.minBy(positionToMinBy, true);
+ }
/**
* Applies an aggregation that that gives the current element with the
@@ -710,6 +724,20 @@ public class DataStream<OUT> {
public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
return this.maxBy(positionToMaxBy, true);
}
+
+ /**
+ * Applies an aggregation that that gives the current element with the
+ * maximum value at the given position, if more elements have the maximum
+ * value at the given position, the operator returns the first one by
+ * default.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(String positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
+ }
/**
* Applies an aggregation that that gives the current element with the
http://git-wip-us.apache.org/repos/asf/flink/blob/40efecf7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
index a2c0f89..18b4b75 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
@@ -47,6 +47,10 @@ public class GroupedDataStream<OUT> extends DataStream<OUT> {
this.keySelector = dataStream.keySelector;
}
+ public KeySelector<OUT, ?> getKeySelector() {
+ return this.keySelector;
+ }
+
/**
* Applies a reduce transformation on the grouped data stream grouped on by
* the given key position. The {@link ReduceFunction} will receive input
http://git-wip-us.apache.org/repos/asf/flink/blob/40efecf7/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 711ce7c..b10bdc6 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -17,15 +17,28 @@
*/
package org.apache.flink.api.scala.streaming
+import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.api.scala.ClosureCleaner
import org.apache.flink.api.common.typeinfo.TypeInformation
import scala.reflect.ClassTag
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.streaming.api.invokable.operator.MapInvokable
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator
+import org.apache.flink.util.Collector
+import org.apache.flink.api.common.functions.FlatMapFunction
+import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.streaming.api.invokable.StreamInvokable
+import org.apache.flink.streaming.api.datastream.GroupedDataStream
+import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable
+import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable
+import org.apache.flink.streaming.api.datastream.GroupedDataStream
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.common.functions.FilterFunction
-class DataStream[OUT](javaStream: JavaStream[OUT]) {
+class DataStream[T](javaStream: JavaStream[T]) {
/* This code is originally from the Apache Spark project. */
/**
@@ -46,29 +59,233 @@ class DataStream[OUT](javaStream: JavaStream[OUT]) {
}
/**
+ * Gets the underlying java DataStream object.
+ */
+ private[flink] def getJavaStream: JavaStream[T] = javaStream
+
+ /**
+ * Sets the degree of parallelism of this operation. This must be greater than 1.
+ */
+ def setParallelism(dop: Int) = {
+ javaStream match {
+ case ds: SingleOutputStreamOperator[_, _] => ds.setParallelism(dop)
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " cannot have " +
+ "parallelism.")
+ }
+ this
+ }
+
+ /**
+ * Returns the degree of parallelism of this operation.
+ */
+ def getParallelism: Int = javaStream match {
+ case op: SingleOutputStreamOperator[_, _] => op.getParallelism
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " does not have " +
+ "parallelism.")
+ }
+
+ def merge(dataStreams: DataStream[T]*): DataStream[T] =
+ new DataStream[T](javaStream.merge(dataStreams.map(_.getJavaStream): _*))
+
+ def groupBy(fields: Int*): DataStream[T] =
+ new DataStream[T](javaStream.groupBy(fields: _*))
+
+ def groupBy(firstField: String, otherFields: String*): DataStream[T] =
+ new DataStream[T](javaStream.groupBy(firstField +: otherFields.toArray: _*))
+
+ def groupBy[K: TypeInformation](fun: T => K): DataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ new DataStream[T](javaStream.groupBy(keyExtractor))
+ }
+
+ def partitionBy(fields: Int*): DataStream[T] =
+ new DataStream[T](javaStream.partitionBy(fields: _*))
+
+ def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
+ new DataStream[T](javaStream.partitionBy(firstField +: otherFields.toArray: _*))
+
+ def partitionBy[K: TypeInformation](fun: T => K): DataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ new DataStream[T](javaStream.partitionBy(keyExtractor))
+ }
+
+ def broadcast: DataStream[T] = new DataStream[T](javaStream.broadcast())
+
+ def shuffle: DataStream[T] = new DataStream[T](javaStream.shuffle())
+
+ def forward: DataStream[T] = new DataStream[T](javaStream.forward())
+
+ def distribute: DataStream[T] = new DataStream[T](javaStream.distribute())
+
+ def max(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.max(field))
+ case field: String => return new DataStream[T](javaStream.max(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ def min(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.min(field))
+ case field: String => return new DataStream[T](javaStream.min(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ def sum(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.sum(field))
+ case field: String => return new DataStream[T](javaStream.sum(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ def maxBy(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.maxBy(field))
+ case field: String => return new DataStream[T](javaStream.maxBy(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ def minBy(field: Any): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.minBy(field))
+ case field: String => return new DataStream[T](javaStream.minBy(field))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ def minBy(field: Any, first: Boolean): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.minBy(field, first))
+ case field: String => return new DataStream[T](javaStream.minBy(field, first))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ def maxBy(field: Any, first: Boolean): DataStream[T] = field match {
+ case field: Int => return new DataStream[T](javaStream.maxBy(field, first))
+ case field: String => return new DataStream[T](javaStream.maxBy(field, first))
+ case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ }
+
+ def count: DataStream[java.lang.Long] = new DataStream[java.lang.Long](javaStream.count())
+
+ /**
* Creates a new DataStream by applying the given function to every element of this DataStream.
*/
- def map[R: TypeInformation: ClassTag](fun: OUT => R): DataStream[R] = {
+ def map[R: TypeInformation: ClassTag](fun: T => R): DataStream[R] = {
if (fun == null) {
throw new NullPointerException("Map function must not be null.")
}
- val mapper = new MapFunction[OUT, R] {
+ val mapper = new MapFunction[T, R] {
val cleanFun = clean(fun)
- def map(in: OUT): R = cleanFun(in)
+ def map(in: T): R = cleanFun(in)
}
- new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[OUT, R](mapper)))
+ new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper)))
}
/**
* Creates a new DataStream by applying the given function to every element of this DataStream.
*/
- def map[R: TypeInformation: ClassTag](mapper: MapFunction[OUT, R]): DataStream[R] = {
+ def map[R: TypeInformation: ClassTag](mapper: MapFunction[T, R]): DataStream[R] = {
if (mapper == null) {
throw new NullPointerException("Map function must not be null.")
}
- new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[OUT, R](mapper)))
+ new DataStream(javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper)))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](flatMapper: FlatMapFunction[T, R]): DataStream[R] = {
+ if (flatMapper == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ new DataStream[R](javaStream.transform("flatMap", implicitly[TypeInformation[R]], new FlatMapInvokable[T, R](flatMapper)))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun: (T, Collector[R]) => Unit): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ val flatMapper = new FlatMapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def flatMap(in: T, out: Collector[R]) { cleanFun(in, out) }
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun: T => TraversableOnce[R]): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ val flatMapper = new FlatMapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def flatMap(in: T, out: Collector[R]) { cleanFun(in) foreach out.collect }
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * Creates a new [[DataStream]] by merging the elements of this DataStream using an associative reduce
+ * function.
+ */
+ def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
+ if (reducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ javaStream match {
+ case ds: GroupedDataStream[_] => new DataStream[T](javaStream.transform("reduce", javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector())))
+ case _ => new DataStream[T](javaStream.transform("reduce", javaStream.getType(), new StreamReduceInvokable[T](reducer)))
+ }
+ }
+
+ /**
+ * Creates a new [[DataStream]] by merging the elements of this DataStream using an associative reduce
+ * function.
+ */
+ def reduce(fun: (T, T) => T): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ val reducer = new ReduceFunction[T] {
+ val cleanFun = clean(fun)
+ def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Creates a new DataSet that contains only the elements satisfying the given filter predicate.
+ */
+ def filter(filter: FilterFunction[T]): DataStream[T] = {
+ if (filter == null) {
+ throw new NullPointerException("Filter function must not be null.")
+ }
+ new DataStream[T](javaStream.filter(filter))
+ }
+
+ def filter(fun: T => Boolean): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Filter function must not be null.")
+ }
+ val filter = new FilterFunction[T] {
+ val cleanFun = clean(fun)
+ def filter(in: T) = cleanFun(in)
+ }
+ this.filter(filter)
}
def print() = javaStream.print()
[34/36] flink git commit: [scala] [streaming] added scala streams as
sources in streaming-api scala examples
Posted by mb...@apache.org.
[scala] [streaming] added scala streams as sources in streaming-api scala examples
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b2f6df7a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b2f6df7a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b2f6df7a
Branch: refs/heads/release-0.8
Commit: b2f6df7aa23914409b9b0c753297f501b67ae028
Parents: 544b924
Author: carbone <se...@gmail.com>
Authored: Tue Jan 6 13:53:01 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 15:48:05 2015 +0100
----------------------------------------------------------------------
.../examples/windowing/TopSpeedWindowing.scala | 44 ++++++++++----------
.../scala/examples/windowing/WindowJoin.scala | 36 ++++++++--------
2 files changed, 40 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b2f6df7a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
index a86b953..3cf5e50 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -21,15 +21,13 @@ package org.apache.flink.streaming.scala.examples.windowing
import java.util.concurrent.TimeUnit._
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.util.Collector
-import scala.math.{max, min}
-
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.streaming.api.scala.windowing.{Delta, Time}
+import org.apache.flink.api.scala._
+import scala.Stream._
+import scala.math._
import scala.util.Random
-import org.apache.flink.streaming.api.scala.windowing.Time
-import org.apache.flink.streaming.api.scala.windowing.Delta
-
/**
* An example of grouped stream windowing where different eviction and
* trigger policies can be used. A source fetches events from cars
@@ -40,7 +38,7 @@ import org.apache.flink.streaming.api.scala.windowing.Delta
*/
object TopSpeedWindowing {
- case class CarSpeed(carId: Int, speed: Int, distance: Double, time: Long)
+ case class CarEvent(carId: Int, speed: Int, distance: Double, time: Long) extends Serializable
def main(args: Array[String]) {
if (!parseParameters(args)) {
@@ -48,10 +46,11 @@ object TopSpeedWindowing {
}
val env = StreamExecutionEnvironment.getExecutionEnvironment
- val cars = env.addSource(carSource _).groupBy("carId")
+ val cars = env.fromCollection(genCarStream())
+ .groupBy("carId")
.window(Time.of(evictionSec, SECONDS))
- .every(Delta.of[CarSpeed](triggerMeters,
- (oldSp,newSp) => newSp.distance-oldSp.distance, CarSpeed(0,0,0,0)))
+ .every(Delta.of[CarEvent](triggerMeters,
+ (oldSp,newSp) => newSp.distance-oldSp.distance, CarEvent(0,0,0,0)))
.reduce((x, y) => if (x.speed > y.speed) x else y)
cars print
@@ -60,19 +59,20 @@ object TopSpeedWindowing {
}
- def carSource(out: Collector[CarSpeed]) = {
-
- val speeds = new Array[Int](numOfCars)
- val distances = new Array[Double](numOfCars)
+ def genCarStream(): Stream[CarEvent] = {
- while (true) {
- Thread sleep 1000
- for (i <- 0 until speeds.length) {
- speeds(i) = if (Random.nextBoolean) min(100, speeds(i) + 5) else max(0, speeds(i) - 5)
- distances(i) += speeds(i) / 3.6d
- out.collect(new CarSpeed(i, speeds(i), distances(i), System.currentTimeMillis))
- }
+ def nextSpeed(carEvent : CarEvent) : CarEvent =
+ {
+ val next =
+ if (Random.nextBoolean) min(100, carEvent.speed + 5) else max(0, carEvent.speed - 5)
+ CarEvent(carEvent.carId, next, carEvent.distance + next/3.6d,System.currentTimeMillis)
+ }
+ def carStream(speeds : Stream[CarEvent]) : Stream[CarEvent] =
+ {
+ Thread.sleep(1000)
+ speeds.append(carStream(speeds.map(nextSpeed)))
}
+ carStream(range(0, numOfCars).map(CarEvent(_,50,0,System.currentTimeMillis())))
}
def parseParameters(args: Array[String]): Boolean = {
http://git-wip-us.apache.org/repos/asf/flink/blob/b2f6df7a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
index da69774..e5dc708 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
@@ -18,8 +18,10 @@
package org.apache.flink.streaming.scala.examples.windowing
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.util.Collector
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+
+import scala.Stream._
import scala.util.Random
import java.util.concurrent.TimeUnit
@@ -34,8 +36,8 @@ object WindowJoin {
val env = StreamExecutionEnvironment.getExecutionEnvironment
//Create streams for names and ages by mapping the inputs to the corresponding objects
- val names = env.addSource(nameStream _).map(x => Name(x._1, x._2))
- val ages = env.addSource(ageStream _).map(x => Age(x._1, x._2))
+ val names = env.fromCollection(nameStream).map(x => Name(x._1, x._2))
+ val ages = env.fromCollection(ageStream).map(x => Age(x._1, x._2))
//Join the two input streams by id on the last second every 2 seconds and create new
//Person objects containing both name and age
@@ -48,24 +50,22 @@ object WindowJoin {
env.execute("WindowJoin")
}
- //Stream source for generating (id, name) pairs
- def nameStream(out: Collector[(Long, String)]) = {
- val names = Array("tom", "jerry", "alice", "bob", "john", "grace")
-
- for (i <- 1 to 10000) {
- if (i % 100 == 0) Thread.sleep(1000) else {
- out.collect((i, names(Random.nextInt(names.length))))
- }
+ def nameStream() : Stream[(Long,String)] = {
+ def nameMapper(names: Array[String])(x: Int) : (Long, String) =
+ {
+ if(x%100==0) Thread.sleep(1000)
+ (x, names(Random.nextInt(names.length)))
}
+ range(1,10000).map(nameMapper(Array("tom", "jerry", "alice", "bob", "john", "grace")))
}
- //Stream source for generating (id, age) pairs
- def ageStream(out: Collector[(Long, Int)]) = {
- for (i <- 1 to 10000) {
- if (i % 100 == 0) Thread.sleep(1000) else {
- out.collect((i, Random.nextInt(90)))
- }
+ def ageStream() : Stream[(Long,Int)] = {
+ def ageMapper(x: Int) : (Long, Int) =
+ {
+ if(x%100==0) Thread.sleep(1000)
+ (x, Random.nextInt(90))
}
+ range(1,10000).map(ageMapper)
}
}
[20/36] flink git commit: [scala] [streaming] added initial
ConnectedDataStream support (no windowing, batching) for the scala api
Posted by mb...@apache.org.
[scala] [streaming] added initial ConnectedDataStream support (no windowing,batching) for the scala api
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/b094d49a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/b094d49a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/b094d49a
Branch: refs/heads/release-0.8
Commit: b094d49a3a0ac2d53698837f215e69372f2f963f
Parents: e1fa9b5
Author: carbone <se...@gmail.com>
Authored: Sat Jan 3 19:46:22 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:07:08 2015 +0100
----------------------------------------------------------------------
.../scala/streaming/ConnectedDataStream.scala | 369 +++++++++++++++++++
1 file changed, 369 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/b094d49a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
new file mode 100644
index 0000000..1c3f5a1
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming
+
+import java.util
+
+import scala.collection.JavaConversions._
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment._
+import org.apache.flink.streaming.api.datastream.{ConnectedDataStream => JavaCStream}
+import org.apache.flink.streaming.api.function.co.{CoWindowFunction, CoFlatMapFunction, CoMapFunction, CoReduceFunction}
+import org.apache.flink.streaming.api.invokable.operator.co.{CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable}
+import org.apache.flink.util.Collector
+
+
+import scala.reflect.ClassTag
+
+class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
+
+ /**
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * @param fun1 for each element of the first input and
+ * @param fun2 for each element of the second input. Each
+ * CoMapFunction call returns exactly one element.
+ *
+ * The CoMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def map[R: TypeInformation: ClassTag](fun1: IN1 => R, fun2: IN2 => R): DataStream[R] = {
+ if (fun1 == null || fun2 == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+ val comapper = new CoMapFunction[IN1,IN2,R] {
+ def map1(in1: IN1): R = clean(fun1)(in1)
+ def map2(in2: IN2): R = clean(fun2)(in2)
+ }
+
+ new DataStream(javaStream.addCoFunction("map",implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1,IN2,R](comapper)))
+ }
+
+ /**
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * {@link CoMapFunction#map1} for each element of the first input and
+ * {@link CoMapFunction#map2} for each element of the second input. Each
+ * CoMapFunction call returns exactly one element. The user can also extend
+ * {@link RichCoMapFunction} to gain access to other features provided by
+ * the {@link RichFuntion} interface.
+ *
+ * @param coMapper
+ * The CoMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def map[R: TypeInformation: ClassTag](coMapper: CoMapFunction[IN1,IN2,R]): DataStream[R] = {
+ if (coMapper == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+
+ new DataStream(javaStream.addCoFunction("map",implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1,IN2,R](coMapper)))
+ }
+
+ /**
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * {@link CoFlatMapFunction#flatMap1} for each element of the first input
+ * and {@link CoFlatMapFunction#flatMap2} for each element of the second
+ * input. Each CoFlatMapFunction call returns any number of elements
+ * including none. The user can also extend {@link RichFlatMapFunction} to
+ * gain access to other features provided by the {@link RichFuntion}
+ * interface.
+ *
+ * @param coFlatMapper
+ * The CoFlatMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def flatMap[R: TypeInformation: ClassTag](coFlatMapper: CoFlatMapFunction[IN1,IN2,R]): DataStream[R] = {
+ if (coFlatMapper == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ new DataStream[R](javaStream.addCoFunction("flatMap", implicitly[TypeInformation[R]],
+ new CoFlatMapInvokable[IN1,IN2, R](coFlatMapper)))
+ }
+
+ /**
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * @param fun1 for each element of the first input
+ * and @param fun2 for each element of the second
+ * input. Each CoFlatMapFunction call returns any number of elements
+ * including none.
+
+ * @return The transformed { @link DataStream}
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun1: (IN1, Collector[R]) => Unit, fun2: (IN2, Collector[R]) => Unit): DataStream[R] = {
+ if (fun1 == null || fun2 == null) {
+ throw new NullPointerException("FlatMap functions must not be null.")
+ }
+ val flatMapper = new CoFlatMapFunction[IN1,IN2, R] {
+ def flatMap1(value: IN1, out: Collector[R]): Unit = clean(fun1)(value,out)
+ def flatMap2(value: IN2, out: Collector[R]): Unit = clean(fun2)(value,out)
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPosition1 and keyPosition2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPosition1
+ * The field used to compute the hashcode of the elements in the
+ * first input stream.
+ * @param keyPosition2
+ * The field used to compute the hashcode of the elements in the
+ * second input stream.
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy(keyPosition1 : Int,keyPosition2: Int) : ConnectedDataStream[IN1,IN2] = {
+ new ConnectedDataStream[IN1,IN2](javaStream.groupBy(keyPosition1,keyPosition2))
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPositions1 and keyPositions2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPositions1
+ * The fields used to group the first input stream.
+ * @param keyPositions2
+ * The fields used to group the second input stream.
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy(keyPositions1 : Array[Int],keyPositions2: Array[Int]) : ConnectedDataStream[IN1,IN2] = {
+ new ConnectedDataStream[IN1,IN2](javaStream.groupBy(keyPositions1,keyPositions2))
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to field1 and field2. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field1
+ * The grouping expression for the first input
+ * @param field2
+ * The grouping expression for the second input
+ * @return The grouped { @link ConnectedDataStream}
+ */
+ def groupBy(field1 : String, field2: String) : ConnectedDataStream[IN1,IN2] = {
+ new ConnectedDataStream[IN1,IN2](javaStream.groupBy(field1,field2))
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to fields1 and fields2. A
+ * field expression is either the name of a public field or a getter method
+ * with parentheses of the {@link DataStream}S underlying type. A dot can be
+ * used to drill down into objects, as in {@code "field1.getInnerField2()" }
+ * .
+ *
+ * @param fields1
+ * The grouping expressions for the first input
+ * @param fields2
+ * The grouping expressions for the second input
+ * @return The grouped { @link ConnectedDataStream}
+ */
+ def groupBy(fields1 : Array[String],fields2: Array[String]) : ConnectedDataStream[IN1,IN2] = {
+ new ConnectedDataStream[IN1,IN2](javaStream.groupBy(fields1,fields2))
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 using fun1 and fun2. Used for applying
+ * function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param fun1
+ * The function used for grouping the first input
+ * @param fun2
+ * The function used for grouping the second input
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy[K: TypeInformation](fun1: IN1 => _, fun2: IN2 => _): ConnectedDataStream[IN1,IN2] = {
+
+ val keyExtractor1 = new KeySelector[IN1, Any] {
+ def getKey(in: IN1) = clean(fun1)(in)
+ }
+ val keyExtractor2 = new KeySelector[IN2, Any] {
+ def getKey(in: IN2) = clean(fun2)(in)
+ }
+
+ new ConnectedDataStream[IN1,IN2](javaStream.groupBy(keyExtractor1,keyExtractor2))
+ }
+
+ /**
+ * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
+ * the outputs to a common type. If the {@link ConnectedDataStream} is
+ * batched or windowed then the reduce transformation is applied on every
+ * sliding batch/window of the data stream. If the connected data stream is
+ * grouped then the reducer is applied on every group of elements sharing
+ * the same key. This type of reduce is much faster than reduceGroup since
+ * the reduce function can be applied incrementally.
+ *
+ * @param coReducer
+ * The { @link CoReduceFunction} that will be called for every
+ * element of the inputs.
+ * @return The transformed { @link DataStream}.
+ */
+ def reduce[R: TypeInformation: ClassTag](coReducer: CoReduceFunction[IN1,IN2,R]): DataStream[R] = {
+ if (coReducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+
+ new DataStream[R](javaStream.addCoFunction("coReduce", implicitly[TypeInformation[R]],
+ new CoReduceInvokable[IN1,IN2,R](coReducer)))
+ }
+
+ /**
+ * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
+ * the outputs to a common type. If the {@link ConnectedDataStream} is
+ * batched or windowed then the reduce transformation is applied on every
+ * sliding batch/window of the data stream. If the connected data stream is
+ * grouped then the reducer is applied on every group of elements sharing
+ * the same key. This type of reduce is much faster than reduceGroup since
+ * the reduce function can be applied incrementally.
+ *
+ * @param reducer1
+ * @param reducer2
+ * @param mapper1
+ * @param mapper2
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def reduce[R: TypeInformation: ClassTag](reducer1: (IN1,IN1) => IN1, reducer2: (IN2,IN2) => IN2,
+ mapper1: IN1 => R, mapper2: IN2 => R): DataStream[R] = {
+ if (mapper1 == null || mapper2 == null) {
+ throw new NullPointerException("Map functions must not be null.")
+ }
+ if (reducer1 == null || reducer2 == null) {
+ throw new NullPointerException("Reduce functions must not be null.")
+ }
+
+ val reducer = new CoReduceFunction[IN1,IN2,R] {
+ def reduce1(value1: IN1, value2: IN1): IN1 = clean(reducer1)(value1,value2)
+ def map2(value: IN2): R = clean(mapper2)(value)
+ def reduce2(value1: IN2, value2: IN2): IN2 = clean(reducer2)(value1,value2)
+ def map1(value: IN1): R = clean(mapper1)(value)
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for for
+ * time aligned windows of the two data streams. System time is used as
+ * default to compute windows.
+ *
+ * @param coWindowFunction
+ * The { @link CoWindowFunction} that will be applied for the time
+ * windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def windowReduce[R: TypeInformation: ClassTag](coWindowFunction: CoWindowFunction[IN1,IN2,R], windowSize:Long, slideInterval: Long) = {
+ if(coWindowFunction == null){
+ throw new NullPointerException("CoWindow function must no be null")
+ }
+
+ new DataStream[R](javaStream.windowReduce(coWindowFunction, windowSize, slideInterval))
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for for
+ * time aligned windows of the two data streams. System time is used as
+ * default to compute windows.
+ *
+ * @param coWindower
+ * The coWindowing function to be applied for the time windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def windowReduce[R: TypeInformation: ClassTag](coWindower: (Seq[IN1], Seq[IN2], Collector[R]) => Unit , windowSize:Long, slideInterval: Long) = {
+ if(coWindower == null){
+ throw new NullPointerException("CoWindow function must no be null")
+ }
+
+ val coWindowFun = new CoWindowFunction[IN1,IN2,R] {
+ def coWindow(first: util.List[IN1], second: util.List[IN2], out: Collector[R]): Unit = clean(coWindower)(first,second,out)
+ }
+
+ new DataStream[R](javaStream.windowReduce(coWindowFun, windowSize, slideInterval))
+ }
+
+ /**
+ * Returns the first {@link DataStream}.
+ *
+ * @return The first DataStream.
+ */
+ def getFirst(): DataStream[IN1] = {
+ new DataStream[IN1](javaStream.getFirst)
+ }
+
+ /**
+ * Returns the second {@link DataStream}.
+ *
+ * @return The second DataStream.
+ */
+ def getSecond(): DataStream[IN2] = {
+ new DataStream[IN2](javaStream.getSecond)
+ }
+
+ /**
+ * Gets the type of the first input
+ *
+ * @return The type of the first input
+ */
+ def getInputType1(): TypeInformation[IN1] = {
+ javaStream.getInputType1
+ }
+
+ /**
+ * Gets the type of the second input
+ *
+ * @return The type of the second input
+ */
+ def getInputType2(): TypeInformation[IN2] = {
+ javaStream.getInputType2
+ }
+
+
+}
[11/36] flink git commit: [scala] [streaming] Modified aggregations
to work on scala tuples
Posted by mb...@apache.org.
[scala] [streaming] Modified aggregations to work on scala tuples
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f7291ea1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f7291ea1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f7291ea1
Branch: refs/heads/release-0.8
Commit: f7291ea1c9fa4a0484ab6bc13e4a594ff6d7c2d2
Parents: 40a3b6b
Author: Gyula Fora <gy...@apache.org>
Authored: Sat Dec 20 23:46:35 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:57:44 2015 +0100
----------------------------------------------------------------------
.../aggregation/AggregationFunction.java | 2 +-
.../aggregation/ComparableAggregator.java | 8 +-
.../api/function/aggregation/SumFunction.java | 12 +-
.../operator/StreamReduceInvokable.java | 1 +
.../streaming/ScalaStreamingAggregator.java | 111 +++++++++++++++++++
.../flink/api/scala/streaming/DataStream.scala | 52 +++++----
.../scala/streaming/WindowedDataStream.scala | 64 ++++-------
7 files changed, 171 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/f7291ea1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
index d95c37e..1c273d3 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/AggregationFunction.java
@@ -22,7 +22,7 @@ import org.apache.flink.api.common.functions.ReduceFunction;
public abstract class AggregationFunction<T> implements ReduceFunction<T> {
private static final long serialVersionUID = 1L;
- int position;
+ public int position;
public AggregationFunction(int pos) {
this.position = pos;
http://git-wip-us.apache.org/repos/asf/flink/blob/f7291ea1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
index 6e2a400..5fb8f62 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
@@ -35,11 +35,11 @@ public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
private static final long serialVersionUID = 1L;
- Comparator comparator;
- boolean byAggregate;
- boolean first;
+ public Comparator comparator;
+ public boolean byAggregate;
+ public boolean first;
- private ComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
+ public ComparableAggregator(int pos, AggregationType aggregationType, boolean first) {
super(pos);
this.comparator = Comparator.getForAggregation(aggregationType);
this.byAggregate = (aggregationType == AggregationType.MAXBY)
http://git-wip-us.apache.org/repos/asf/flink/blob/f7291ea1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
index 1ac236d..2aef19c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumFunction.java
@@ -45,7 +45,7 @@ public abstract class SumFunction implements Serializable{
}
}
- private static class IntSum extends SumFunction {
+ public static class IntSum extends SumFunction {
private static final long serialVersionUID = 1L;
@Override
@@ -54,7 +54,7 @@ public abstract class SumFunction implements Serializable{
}
}
- private static class LongSum extends SumFunction {
+ public static class LongSum extends SumFunction {
private static final long serialVersionUID = 1L;
@Override
@@ -63,7 +63,7 @@ public abstract class SumFunction implements Serializable{
}
}
- private static class DoubleSum extends SumFunction {
+ public static class DoubleSum extends SumFunction {
private static final long serialVersionUID = 1L;
@@ -73,7 +73,7 @@ public abstract class SumFunction implements Serializable{
}
}
- private static class ShortSum extends SumFunction {
+ public static class ShortSum extends SumFunction {
private static final long serialVersionUID = 1L;
@Override
@@ -82,7 +82,7 @@ public abstract class SumFunction implements Serializable{
}
}
- private static class FloatSum extends SumFunction {
+ public static class FloatSum extends SumFunction {
private static final long serialVersionUID = 1L;
@Override
@@ -91,7 +91,7 @@ public abstract class SumFunction implements Serializable{
}
}
- private static class ByteSum extends SumFunction {
+ public static class ByteSum extends SumFunction {
private static final long serialVersionUID = 1L;
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/f7291ea1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
index 4bb78b8..5f5cb12 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
@@ -52,6 +52,7 @@ public class StreamReduceInvokable<IN> extends StreamInvokable<IN, IN> {
currentValue = reducer.reduce(currentValue, nextValue);
} else {
currentValue = nextValue;
+
}
collector.collect(currentValue);
http://git-wip-us.apache.org/repos/asf/flink/blob/f7291ea1/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java b/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java
new file mode 100644
index 0000000..2f587d7
--- /dev/null
+++ b/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.scala.streaming;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
+import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
+import org.apache.flink.streaming.api.function.aggregation.SumFunction;
+
+import scala.Product;
+
+public class ScalaStreamingAggregator<IN extends Product> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ TupleSerializerBase<IN> serializer;
+ Object[] fields;
+ int length;
+ int position;
+
+ public ScalaStreamingAggregator(TypeSerializer<IN> serializer, int pos) {
+ this.serializer = (TupleSerializerBase<IN>) serializer;
+ this.length = this.serializer.getArity();
+ this.fields = new Object[this.length];
+ this.position = pos;
+ }
+
+ public class Sum extends AggregationFunction<IN> {
+ private static final long serialVersionUID = 1L;
+ SumFunction sumFunction;
+
+ public Sum(SumFunction func) {
+ super(ScalaStreamingAggregator.this.position);
+ this.sumFunction = func;
+ }
+
+ @Override
+ public IN reduce(IN value1, IN value2) throws Exception {
+ for (int i = 0; i < length; i++) {
+ fields[i] = value2.productElement(i);
+ }
+
+ fields[position] = sumFunction.add(fields[position], value1.productElement(position));
+
+ return serializer.createInstance(fields);
+ }
+ }
+
+ public class ProductComparableAggregator extends ComparableAggregator<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ public ProductComparableAggregator(AggregationFunction.AggregationType aggregationType,
+ boolean first) {
+ super(ScalaStreamingAggregator.this.position, aggregationType, first);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public IN reduce(IN value1, IN value2) throws Exception {
+ Object v1 = value1.productElement(position);
+ Object v2 = value2.productElement(position);
+
+ int c = comparator.isExtremal((Comparable<Object>) v1, v2);
+
+ if (byAggregate) {
+ if (c == 1) {
+ return value1;
+ }
+ if (first) {
+ if (c == 0) {
+ return value1;
+ }
+ }
+
+ return value2;
+ } else {
+ for (int i = 0; i < length; i++) {
+ fields[i] = value2.productElement(i);
+ }
+
+ if (c == 1) {
+ fields[position] = v1;
+ }
+
+ return serializer.createInstance(fields);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/f7291ea1/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 42ec709..ecf5615 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -44,6 +44,11 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy
import org.apache.flink.streaming.api.collector.OutputSelector
import scala.collection.JavaConversions._
import java.util.HashMap
+import org.apache.flink.streaming.api.function.aggregation.SumFunction
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
+import com.amazonaws.services.cloudfront_2012_03_15.model.InvalidArgumentException
class DataStream[T](javaStream: JavaStream[T]) {
@@ -230,53 +235,52 @@ class DataStream[T](javaStream: JavaStream[T]) {
* the given position.
*
*/
- def max(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.max(field))
- case field: String => return new DataStream[T](javaStream.max(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
/**
* Applies an aggregation that that gives the current minimum of the data stream at
* the given position.
*
*/
- def min(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.min(field))
- case field: String => return new DataStream[T](javaStream.min(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
/**
* Applies an aggregation that sums the data stream at the given position.
*
*/
- def sum(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.sum(field))
- case field: String => return new DataStream[T](javaStream.sum(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
/**
* Applies an aggregation that that gives the current minimum element of the data stream by
* the given position. When equality, the user can set to get the first or last element with the minimal value.
*
*/
- def minBy(field: Any, first: Boolean = true): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.minBy(field, first))
- case field: String => return new DataStream[T](javaStream.minBy(field, first))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MINBY, position, first)
/**
* Applies an aggregation that that gives the current maximum element of the data stream by
* the given position. When equality, the user can set to get the first or last element with the maximal value.
*
*/
- def maxBy(field: Any, first: Boolean = true): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.maxBy(field, first))
- case field: String => return new DataStream[T](javaStream.maxBy(field, first))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ def maxBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MAXBY, position, first)
+
+ private def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true): DataStream[T] = {
+
+ val jStream = javaStream.asInstanceOf[JavaStream[Product]]
+ val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
+
+ val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
+
+ val reducer = aggregationType match {
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).getTypeClass()));
+ case _ => new agg.ProductComparableAggregator(aggregationType, first)
+ }
+
+ val invokable = jStream match {
+ case groupedStream: GroupedDataStream[_] => new GroupedReduceInvokable(reducer, groupedStream.getKeySelector())
+ case _ => new StreamReduceInvokable(reducer)
+ }
+ new DataStream[Product](jStream.transform("aggregation", jStream.getType(), invokable)).asInstanceOf[DataStream[T]]
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/f7291ea1/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
index c686497..c037305 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/WindowedDataStream.scala
@@ -36,6 +36,9 @@ import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
import org.apache.flink.api.common.functions.GroupReduceFunction
import org.apache.flink.streaming.api.invokable.StreamInvokable
import scala.collection.JavaConversions._
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.streaming.api.function.aggregation.SumFunction
class WindowedDataStream[T](javaStream: JavaWStream[T]) {
@@ -158,75 +161,48 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
* the given position.
*
*/
- def max(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.max(field))
- case field: String => return new DataStream[T](javaStream.max(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
/**
* Applies an aggregation that that gives the minimum of the elements in the window at
* the given position.
*
*/
- def min(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.min(field))
- case field: String => return new DataStream[T](javaStream.min(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
/**
* Applies an aggregation that sums the elements in the window at the given position.
*
*/
- def sum(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.sum(field))
- case field: String => return new DataStream[T](javaStream.sum(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
/**
* Applies an aggregation that that gives the maximum element of the window by
* the given position. When equality, returns the first.
*
*/
- def maxBy(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.maxBy(field))
- case field: String => return new DataStream[T](javaStream.maxBy(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def maxBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MAXBY, position, first)
/**
* Applies an aggregation that that gives the minimum element of the window by
* the given position. When equality, returns the first.
*
*/
- def minBy(field: Any): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.minBy(field))
- case field: String => return new DataStream[T](javaStream.minBy(field))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MINBY, position, first)
- /**
- * Applies an aggregation that that gives the minimum element of the window by
- * the given position. When equality, the user can set to get the first or last element with the minimal value.
- *
- */
- def minBy(field: Any, first: Boolean): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.minBy(field, first))
- case field: String => return new DataStream[T](javaStream.minBy(field, first))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
- }
+ def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true): DataStream[T] = {
- /**
- * Applies an aggregation that that gives the maximum element of the window by
- * the given position. When equality, the user can set to get the first or last element with the maximal value.
- *
- */
- def maxBy(field: Any, first: Boolean): DataStream[T] = field match {
- case field: Int => return new DataStream[T](javaStream.maxBy(field, first))
- case field: String => return new DataStream[T](javaStream.maxBy(field, first))
- case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
+ val jStream = javaStream.asInstanceOf[JavaWStream[Product]]
+ val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
+
+ val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
+
+ val reducer = aggregationType match {
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).getTypeClass()));
+ case _ => new agg.ProductComparableAggregator(aggregationType, first)
+ }
+
+ new DataStream[Product](jStream.reduce(reducer)).asInstanceOf[DataStream[T]]
}
}
\ No newline at end of file
[21/36] flink git commit: [scala] [streaming] Added connect op to
DataStream and implicit conversion for ConnectedDataStreams
Posted by mb...@apache.org.
[scala] [streaming] Added connect op to DataStream and implicit conversion for ConnectedDataStreams
[scala] [streaming] Changed return types to implicits
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7eaf726a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7eaf726a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7eaf726a
Branch: refs/heads/release-0.8
Commit: 7eaf726aeab1ac461e934a4b9d5f87404a64bc0d
Parents: b094d49
Author: carbone <se...@gmail.com>
Authored: Sat Jan 3 20:00:13 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 18:07:21 2015 +0100
----------------------------------------------------------------------
.../api/datastream/WindowedDataStream.java | 6 +-
.../windowing/TopSpeedWindowingExample.java | 175 ++++++++++---------
.../streaming/windowing/TopSpeedWindowing.scala | 11 +-
.../scala/streaming/ConnectedDataStream.scala | 171 +++++++++---------
.../flink/api/scala/streaming/DataStream.scala | 15 +-
.../scala/streaming/StreamingConversions.scala | 6 +-
6 files changed, 208 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/7eaf726a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 287f29d..e81395d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -231,7 +231,7 @@ public class WindowedDataStream<OUT> {
* @return The transformed DataStream
*/
public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reduceFunction) {
- return dataStream.transform("NextGenWindowReduce", getType(),
+ return dataStream.transform("WindowReduce", getType(),
getReduceInvokable(reduceFunction));
}
@@ -255,7 +255,7 @@ public class WindowedDataStream<OUT> {
TypeInformation<R> outType = TypeExtractor
.getGroupReduceReturnTypes(reduceFunction, inType);
- return dataStream.transform("NextGenWindowReduce", outType,
+ return dataStream.transform("WindowReduce", outType,
getReduceGroupInvokable(reduceFunction));
}
@@ -279,7 +279,7 @@ public class WindowedDataStream<OUT> {
public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
GroupReduceFunction<OUT, R> reduceFunction, TypeInformation<R> outType) {
- return dataStream.transform("NextGenWindowReduce", outType,
+ return dataStream.transform("WindowReduce", outType,
getReduceGroupInvokable(reduceFunction));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7eaf726a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
index bc3bba5..0f5d8eb 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
@@ -17,7 +17,6 @@
package org.apache.flink.streaming.examples.windowing;
-
import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -32,90 +31,98 @@ import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
- * An example of grouped stream windowing where different eviction and trigger policies can be used.
- * A source fetches events from cars every 1 sec containing their id, their current speed (kmh),
- * overall elapsed distance (m) and a timestamp. The streaming
- * example triggers the top speed of each car every x meters elapsed for the last y seconds.
+ * An example of grouped stream windowing where different eviction and trigger
+ * policies can be used. A source fetches events from cars every 1 sec
+ * containing their id, their current speed (kmh), overall elapsed distance (m)
+ * and a timestamp. The streaming example triggers the top speed of each car
+ * every x meters elapsed for the last y seconds.
*/
public class TopSpeedWindowingExample {
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- DataStream topSpeeds = env
- .addSource(CarSource.create(numOfCars))
- .groupBy(0)
- .window(Time.of(evictionSec, TimeUnit.SECONDS))
- .every(Delta.of(
- new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() {
- @Override
- public double getDelta(Tuple4<Integer, Integer, Double, Long> oldDataPoint, Tuple4<Integer, Integer, Double, Long> newDataPoint) {
- return newDataPoint.f2 - oldDataPoint.f2;
- }
- }
- , new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l), triggerMeters))
- .maxBy(1);
-
- topSpeeds.print();
- env.execute("CarTopSpeedWindowingExample");
- }
-
- private static class CarSource implements SourceFunction<Tuple4<Integer, Integer, Double, Long>> {
- private Integer[] speeds;
- private Double[] distances;
-
- private Random rand = new Random();
-
- private CarSource(int numOfCars) {
- speeds = new Integer[numOfCars];
- distances = new Double[numOfCars];
- Arrays.fill(speeds, 50);
- Arrays.fill(distances, 0d);
- }
-
- public static CarSource create(int cars) {
- return new CarSource(cars);
- }
-
- @Override
- public void invoke(Collector<Tuple4<Integer, Integer, Double, Long>> collector) throws Exception {
-
- while (true) {
- Thread.sleep(1000);
- for (int carId = 0; carId < speeds.length; carId++) {
- if (rand.nextBoolean())
- speeds[carId] = Math.min(100, speeds[carId] + 5);
- else
- speeds[carId] = Math.max(0, speeds[carId] - 5);
- distances[carId] += speeds[carId] / 3.6d;
- collector.collect(new Tuple4<Integer, Integer, Double, Long>(carId, speeds[carId], distances[carId], System.currentTimeMillis()));
- }
- }
- }
- }
-
- private static int numOfCars = 2;
- private static int evictionSec = 10;
- private static double triggerMeters = 50;
-
- private static boolean parseParameters(String[] args) {
-
- if (args.length > 0) {
- if (args.length == 3) {
- numOfCars = Integer.valueOf(args[0]);
- evictionSec = Integer.valueOf(args[1]);
- triggerMeters = Double.valueOf(args[2]);
- } else {
- System.err.println("Usage: TopSpeedWindowingExample <numCars> <evictSec> <triggerMeters>");
- return false;
- }
- }
- return true;
- }
+ public static void main(String[] args) throws Exception {
+
+ if (!parseParameters(args)) {
+ return;
+ }
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ @SuppressWarnings({ "rawtypes", "serial" })
+ DataStream topSpeeds = env
+ .addSource(CarSource.create(numOfCars))
+ .groupBy(0)
+ .window(Time.of(evictionSec, TimeUnit.SECONDS))
+ .every(Delta.of(triggerMeters,
+ new DeltaFunction<Tuple4<Integer, Integer, Double, Long>>() {
+ @Override
+ public double getDelta(
+ Tuple4<Integer, Integer, Double, Long> oldDataPoint,
+ Tuple4<Integer, Integer, Double, Long> newDataPoint) {
+ return newDataPoint.f2 - oldDataPoint.f2;
+ }
+ }, new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l))).maxBy(1);
+
+ topSpeeds.print();
+ env.execute("CarTopSpeedWindowingExample");
+ }
+
+ private static class CarSource implements
+ SourceFunction<Tuple4<Integer, Integer, Double, Long>> {
+
+ private static final long serialVersionUID = 1L;
+ private Integer[] speeds;
+ private Double[] distances;
+
+ private Random rand = new Random();
+
+ private CarSource(int numOfCars) {
+ speeds = new Integer[numOfCars];
+ distances = new Double[numOfCars];
+ Arrays.fill(speeds, 50);
+ Arrays.fill(distances, 0d);
+ }
+
+ public static CarSource create(int cars) {
+ return new CarSource(cars);
+ }
+
+ @Override
+ public void invoke(Collector<Tuple4<Integer, Integer, Double, Long>> collector)
+ throws Exception {
+
+ while (true) {
+ Thread.sleep(1000);
+ for (int carId = 0; carId < speeds.length; carId++) {
+ if (rand.nextBoolean()) {
+ speeds[carId] = Math.min(100, speeds[carId] + 5);
+ } else {
+ speeds[carId] = Math.max(0, speeds[carId] - 5);
+ }
+ distances[carId] += speeds[carId] / 3.6d;
+ collector.collect(new Tuple4<Integer, Integer, Double, Long>(carId,
+ speeds[carId], distances[carId], System.currentTimeMillis()));
+ }
+ }
+ }
+ }
+
+ private static int numOfCars = 2;
+ private static int evictionSec = 10;
+ private static double triggerMeters = 50;
+
+ private static boolean parseParameters(String[] args) {
+
+ if (args.length > 0) {
+ if (args.length == 3) {
+ numOfCars = Integer.valueOf(args[0]);
+ evictionSec = Integer.valueOf(args[1]);
+ triggerMeters = Double.valueOf(args[2]);
+ } else {
+ System.err
+ .println("Usage: TopSpeedWindowingExample <numCars> <evictSec> <triggerMeters>");
+ return false;
+ }
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7eaf726a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
index 10ddfd8..e39fb11 100644
--- a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
+++ b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
@@ -30,10 +30,12 @@ import scala.math.{max, min}
import scala.util.Random
/**
- * An example of grouped stream windowing where different eviction and trigger policies can be used.
- * A source fetches events from cars every 1 sec containing their id, their current speed (kmh),
+ * An example of grouped stream windowing where different eviction and
+ * trigger policies can be used.A source fetches events from cars
+ * every 1 sec containing their id, their current speed (kmh),
* overall elapsed distance (m) and a timestamp. The streaming
- * example triggers the top speed of each car every x meters elapsed for the last y seconds.
+ * example triggers the top speed of each car every x meters elapsed
+ * for the last y seconds.
*/
object TopSpeedWindowing {
@@ -47,7 +49,8 @@ object TopSpeedWindowing {
val env = StreamExecutionEnvironment.getExecutionEnvironment
val cars = env.addSource(carSource _).groupBy("carId")
.window(Time.of(evictionSec, SECONDS))
- .every(Delta.of[CarSpeed](triggerMeters, (oldSp,newSp) => newSp.distance-oldSp.distance, CarSpeed(0,0,0,0)))
+ .every(Delta.of[CarSpeed](triggerMeters,
+ (oldSp,newSp) => newSp.distance-oldSp.distance, CarSpeed(0,0,0,0)))
.reduce((x, y) => if (x.speed > y.speed) x else y)
cars print
http://git-wip-us.apache.org/repos/asf/flink/blob/7eaf726a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
index 1c3f5a1..985e512 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
@@ -1,12 +1,13 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -19,19 +20,19 @@ package org.apache.flink.api.scala.streaming
import java.util
-import scala.collection.JavaConversions._
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment._
-import org.apache.flink.streaming.api.datastream.{ConnectedDataStream => JavaCStream}
-import org.apache.flink.streaming.api.function.co.{CoWindowFunction, CoFlatMapFunction, CoMapFunction, CoReduceFunction}
-import org.apache.flink.streaming.api.invokable.operator.co.{CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable}
+import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaCStream }
+import org.apache.flink.streaming.api.function.co.{ CoFlatMapFunction, CoMapFunction, CoReduceFunction, CoWindowFunction }
+import org.apache.flink.streaming.api.invokable.operator.co.{ CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable }
import org.apache.flink.util.Collector
+import org.apache.flink.api.scala.streaming.StreamingConversions._
-
+import scala.collection.JavaConversions._
import scala.reflect.ClassTag
-class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
+class ConnectedDataStream[IN1, IN2](javaStream: JavaCStream[IN1, IN2]) {
/**
* Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
@@ -40,21 +41,22 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* @param fun2 for each element of the second input. Each
* CoMapFunction call returns exactly one element.
*
- * The CoMapFunction used to jointly transform the two input
+ * The CoMapFunction used to jointly transform the two input
* DataStreams
* @return The transformed { @link DataStream}
*/
- def map[R: TypeInformation: ClassTag](fun1: IN1 => R, fun2: IN2 => R): DataStream[R] = {
+ def map[R: TypeInformation: ClassTag](fun1: IN1 => R, fun2: IN2 => R):
+ DataStream[R] = {
if (fun1 == null || fun2 == null) {
throw new NullPointerException("Map function must not be null.")
}
- val comapper = new CoMapFunction[IN1,IN2,R] {
- def map1(in1: IN1): R = clean(fun1)(in1)
- def map2(in2: IN2): R = clean(fun2)(in2)
+ val comapper = new CoMapFunction[IN1, IN2, R] {
+ def map1(in1: IN1): R = clean(fun1)(in1)
+ def map2(in2: IN2): R = clean(fun2)(in2)
}
- new DataStream(javaStream.addCoFunction("map",implicitly[TypeInformation[R]],
- new CoMapInvokable[IN1,IN2,R](comapper)))
+ new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1, IN2, R](comapper)))
}
/**
@@ -67,17 +69,18 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* the {@link RichFuntion} interface.
*
* @param coMapper
- * The CoMapFunction used to jointly transform the two input
+ * The CoMapFunction used to jointly transform the two input
* DataStreams
* @return The transformed { @link DataStream}
*/
- def map[R: TypeInformation: ClassTag](coMapper: CoMapFunction[IN1,IN2,R]): DataStream[R] = {
+ def map[R: TypeInformation: ClassTag](coMapper: CoMapFunction[IN1, IN2, R]):
+ DataStream[R] = {
if (coMapper == null) {
throw new NullPointerException("Map function must not be null.")
}
- new DataStream(javaStream.addCoFunction("map",implicitly[TypeInformation[R]],
- new CoMapInvokable[IN1,IN2,R](coMapper)))
+ new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1, IN2, R](coMapper)))
}
/**
@@ -91,16 +94,17 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* interface.
*
* @param coFlatMapper
- * The CoFlatMapFunction used to jointly transform the two input
+ * The CoFlatMapFunction used to jointly transform the two input
* DataStreams
* @return The transformed { @link DataStream}
*/
- def flatMap[R: TypeInformation: ClassTag](coFlatMapper: CoFlatMapFunction[IN1,IN2,R]): DataStream[R] = {
+ def flatMap[R: TypeInformation: ClassTag](coFlatMapper: CoFlatMapFunction[IN1, IN2, R]):
+ DataStream[R] = {
if (coFlatMapper == null) {
throw new NullPointerException("FlatMap function must not be null.")
}
new DataStream[R](javaStream.addCoFunction("flatMap", implicitly[TypeInformation[R]],
- new CoFlatMapInvokable[IN1,IN2, R](coFlatMapper)))
+ new CoFlatMapInvokable[IN1, IN2, R](coFlatMapper)))
}
/**
@@ -110,16 +114,17 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* and @param fun2 for each element of the second
* input. Each CoFlatMapFunction call returns any number of elements
* including none.
-
+ *
* @return The transformed { @link DataStream}
*/
- def flatMap[R: TypeInformation: ClassTag](fun1: (IN1, Collector[R]) => Unit, fun2: (IN2, Collector[R]) => Unit): DataStream[R] = {
+ def flatMap[R: TypeInformation: ClassTag](fun1: (IN1, Collector[R]) => Unit,
+ fun2: (IN2, Collector[R]) => Unit): DataStream[R] = {
if (fun1 == null || fun2 == null) {
throw new NullPointerException("FlatMap functions must not be null.")
}
- val flatMapper = new CoFlatMapFunction[IN1,IN2, R] {
- def flatMap1(value: IN1, out: Collector[R]): Unit = clean(fun1)(value,out)
- def flatMap2(value: IN2, out: Collector[R]): Unit = clean(fun2)(value,out)
+ val flatMapper = new CoFlatMapFunction[IN1, IN2, R] {
+ def flatMap1(value: IN1, out: Collector[R]): Unit = clean(fun1)(value, out)
+ def flatMap2(value: IN2, out: Collector[R]): Unit = clean(fun2)(value, out)
}
flatMap(flatMapper)
}
@@ -131,15 +136,15 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* {@link ConnectedDataStream#reduce}
*
* @param keyPosition1
- * The field used to compute the hashcode of the elements in the
+ * The field used to compute the hashcode of the elements in the
* first input stream.
* @param keyPosition2
- * The field used to compute the hashcode of the elements in the
+ * The field used to compute the hashcode of the elements in the
* second input stream.
* @return @return The transformed { @link ConnectedDataStream}
*/
- def groupBy(keyPosition1 : Int,keyPosition2: Int) : ConnectedDataStream[IN1,IN2] = {
- new ConnectedDataStream[IN1,IN2](javaStream.groupBy(keyPosition1,keyPosition2))
+ def groupBy(keyPosition1: Int, keyPosition2: Int): ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(keyPosition1, keyPosition2)
}
/**
@@ -149,13 +154,14 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* {@link ConnectedDataStream#reduce}
*
* @param keyPositions1
- * The fields used to group the first input stream.
+ * The fields used to group the first input stream.
* @param keyPositions2
- * The fields used to group the second input stream.
+ * The fields used to group the second input stream.
* @return @return The transformed { @link ConnectedDataStream}
*/
- def groupBy(keyPositions1 : Array[Int],keyPositions2: Array[Int]) : ConnectedDataStream[IN1,IN2] = {
- new ConnectedDataStream[IN1,IN2](javaStream.groupBy(keyPositions1,keyPositions2))
+ def groupBy(keyPositions1: Array[Int], keyPositions2: Array[Int]):
+ ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(keyPositions1, keyPositions2)
}
/**
@@ -166,13 +172,13 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* to drill down into objects, as in {@code "field1.getInnerField2()" }.
*
* @param field1
- * The grouping expression for the first input
+ * The grouping expression for the first input
* @param field2
- * The grouping expression for the second input
+ * The grouping expression for the second input
* @return The grouped { @link ConnectedDataStream}
*/
- def groupBy(field1 : String, field2: String) : ConnectedDataStream[IN1,IN2] = {
- new ConnectedDataStream[IN1,IN2](javaStream.groupBy(field1,field2))
+ def groupBy(field1: String, field2: String): ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(field1, field2)
}
/**
@@ -184,13 +190,14 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* .
*
* @param fields1
- * The grouping expressions for the first input
+ * The grouping expressions for the first input
* @param fields2
- * The grouping expressions for the second input
+ * The grouping expressions for the second input
* @return The grouped { @link ConnectedDataStream}
*/
- def groupBy(fields1 : Array[String],fields2: Array[String]) : ConnectedDataStream[IN1,IN2] = {
- new ConnectedDataStream[IN1,IN2](javaStream.groupBy(fields1,fields2))
+ def groupBy(fields1: Array[String], fields2: Array[String]):
+ ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(fields1, fields2)
}
/**
@@ -200,12 +207,13 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* {@link ConnectedDataStream#reduce}
*
* @param fun1
- * The function used for grouping the first input
+ * The function used for grouping the first input
* @param fun2
- * The function used for grouping the second input
+ * The function used for grouping the second input
* @return @return The transformed { @link ConnectedDataStream}
*/
- def groupBy[K: TypeInformation](fun1: IN1 => _, fun2: IN2 => _): ConnectedDataStream[IN1,IN2] = {
+ def groupBy[K: TypeInformation](fun1: IN1 => _, fun2: IN2 => _):
+ ConnectedDataStream[IN1, IN2] = {
val keyExtractor1 = new KeySelector[IN1, Any] {
def getKey(in: IN1) = clean(fun1)(in)
@@ -214,7 +222,7 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
def getKey(in: IN2) = clean(fun2)(in)
}
- new ConnectedDataStream[IN1,IN2](javaStream.groupBy(keyExtractor1,keyExtractor2))
+ javaStream.groupBy(keyExtractor1, keyExtractor2)
}
/**
@@ -227,17 +235,18 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* the reduce function can be applied incrementally.
*
* @param coReducer
- * The { @link CoReduceFunction} that will be called for every
+ * The { @link CoReduceFunction} that will be called for every
* element of the inputs.
* @return The transformed { @link DataStream}.
*/
- def reduce[R: TypeInformation: ClassTag](coReducer: CoReduceFunction[IN1,IN2,R]): DataStream[R] = {
+ def reduce[R: TypeInformation: ClassTag](coReducer: CoReduceFunction[IN1, IN2, R]):
+ DataStream[R] = {
if (coReducer == null) {
throw new NullPointerException("Reduce function must not be null.")
}
new DataStream[R](javaStream.addCoFunction("coReduce", implicitly[TypeInformation[R]],
- new CoReduceInvokable[IN1,IN2,R](coReducer)))
+ new CoReduceInvokable[IN1, IN2, R](coReducer)))
}
/**
@@ -256,20 +265,20 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
*
* @return The transformed { @link DataStream}.
*/
- def reduce[R: TypeInformation: ClassTag](reducer1: (IN1,IN1) => IN1, reducer2: (IN2,IN2) => IN2,
- mapper1: IN1 => R, mapper2: IN2 => R): DataStream[R] = {
+ def reduce[R: TypeInformation: ClassTag](reducer1: (IN1, IN1) => IN1,
+ reducer2: (IN2, IN2) => IN2,mapper1: IN1 => R, mapper2: IN2 => R): DataStream[R] = {
if (mapper1 == null || mapper2 == null) {
throw new NullPointerException("Map functions must not be null.")
}
if (reducer1 == null || reducer2 == null) {
throw new NullPointerException("Reduce functions must not be null.")
}
-
- val reducer = new CoReduceFunction[IN1,IN2,R] {
- def reduce1(value1: IN1, value2: IN1): IN1 = clean(reducer1)(value1,value2)
- def map2(value: IN2): R = clean(mapper2)(value)
- def reduce2(value1: IN2, value2: IN2): IN2 = clean(reducer2)(value1,value2)
- def map1(value: IN1): R = clean(mapper1)(value)
+
+ val reducer = new CoReduceFunction[IN1, IN2, R] {
+ def reduce1(value1: IN1, value2: IN1): IN1 = clean(reducer1)(value1, value2)
+ def map2(value: IN2): R = clean(mapper2)(value)
+ def reduce2(value1: IN2, value2: IN2): IN2 = clean(reducer2)(value1, value2)
+ def map1(value: IN1): R = clean(mapper1)(value)
}
reduce(reducer)
}
@@ -281,23 +290,24 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* default to compute windows.
*
* @param coWindowFunction
- * The { @link CoWindowFunction} that will be applied for the time
+ * The { @link CoWindowFunction} that will be applied for the time
* windows.
* @param windowSize
- * Size of the windows that will be aligned for both streams in
+ * Size of the windows that will be aligned for both streams in
* milliseconds.
* @param slideInterval
- * After every function call the windows will be slid by this
+ * After every function call the windows will be slid by this
* interval.
*
* @return The transformed { @link DataStream}.
*/
- def windowReduce[R: TypeInformation: ClassTag](coWindowFunction: CoWindowFunction[IN1,IN2,R], windowSize:Long, slideInterval: Long) = {
- if(coWindowFunction == null){
+ def windowReduce[R: TypeInformation: ClassTag](coWindowFunction:
+ CoWindowFunction[IN1, IN2, R], windowSize: Long, slideInterval: Long) = {
+ if (coWindowFunction == null) {
throw new NullPointerException("CoWindow function must no be null")
}
- new DataStream[R](javaStream.windowReduce(coWindowFunction, windowSize, slideInterval))
+ javaStream.windowReduce(coWindowFunction, windowSize, slideInterval)
}
/**
@@ -307,26 +317,28 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* default to compute windows.
*
* @param coWindower
- * The coWindowing function to be applied for the time windows.
+ * The coWindowing function to be applied for the time windows.
* @param windowSize
- * Size of the windows that will be aligned for both streams in
+ * Size of the windows that will be aligned for both streams in
* milliseconds.
* @param slideInterval
- * After every function call the windows will be slid by this
+ * After every function call the windows will be slid by this
* interval.
*
* @return The transformed { @link DataStream}.
*/
- def windowReduce[R: TypeInformation: ClassTag](coWindower: (Seq[IN1], Seq[IN2], Collector[R]) => Unit , windowSize:Long, slideInterval: Long) = {
- if(coWindower == null){
+ def windowReduce[R: TypeInformation: ClassTag](coWindower: (Seq[IN1], Seq[IN2],
+ Collector[R]) => Unit, windowSize: Long, slideInterval: Long) = {
+ if (coWindower == null) {
throw new NullPointerException("CoWindow function must no be null")
}
- val coWindowFun = new CoWindowFunction[IN1,IN2,R] {
- def coWindow(first: util.List[IN1], second: util.List[IN2], out: Collector[R]): Unit = clean(coWindower)(first,second,out)
+ val coWindowFun = new CoWindowFunction[IN1, IN2, R] {
+ def coWindow(first: util.List[IN1], second: util.List[IN2],
+ out: Collector[R]): Unit = clean(coWindower)(first, second, out)
}
- new DataStream[R](javaStream.windowReduce(coWindowFun, windowSize, slideInterval))
+ javaStream.windowReduce(coWindowFun, windowSize, slideInterval)
}
/**
@@ -335,7 +347,7 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* @return The first DataStream.
*/
def getFirst(): DataStream[IN1] = {
- new DataStream[IN1](javaStream.getFirst)
+ javaStream.getFirst
}
/**
@@ -344,7 +356,7 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
* @return The second DataStream.
*/
def getSecond(): DataStream[IN2] = {
- new DataStream[IN2](javaStream.getSecond)
+ javaStream.getSecond
}
/**
@@ -365,5 +377,4 @@ class ConnectedDataStream[IN1,IN2] (javaStream: JavaCStream[IN1,IN2]) {
javaStream.getInputType2
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/7eaf726a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index 546d8a9..ccfd176 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -19,22 +19,20 @@
package org.apache.flink.api.scala.streaming
import org.apache.flink.api.scala._
import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream,
+ SingleOutputStreamOperator, GroupedDataStream}
import org.apache.flink.api.common.typeinfo.TypeInformation
import scala.reflect.ClassTag
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.functions.MapFunction
import org.apache.flink.streaming.api.invokable.operator.MapInvokable
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator
import org.apache.flink.util.Collector
import org.apache.flink.api.common.functions.FlatMapFunction
import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
import org.apache.flink.api.common.functions.ReduceFunction
import org.apache.flink.streaming.api.invokable.StreamInvokable
-import org.apache.flink.streaming.api.datastream.GroupedDataStream
import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable
import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable
-import org.apache.flink.streaming.api.datastream.GroupedDataStream
import org.apache.flink.api.common.functions.ReduceFunction
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.common.functions.FilterFunction
@@ -94,6 +92,15 @@ class DataStream[T](javaStream: JavaStream[T]) {
javaStream.merge(dataStreams.map(_.getJavaStream): _*)
/**
+ * Creates a new ConnectedDataStream by connecting
+ * DataStream outputs of different type with each other. The
+ * DataStreams connected using this operators can be used with CoFunctions.
+ *
+ */
+ def connect[T2](dataStream: DataStream[T2]): ConnectedDataStream[T, T2] =
+ javaStream.connect(dataStream.getJavaStream)
+
+ /**
* Groups the elements of a DataStream by the given key positions (for tuple/array types) to
* be used with grouped operators like grouped reduce or grouped aggregations
*
http://git-wip-us.apache.org/repos/asf/flink/blob/7eaf726a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
index a34d0dc..9aefa04 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamingConversions.scala
@@ -21,9 +21,10 @@ package org.apache.flink.api.scala.streaming
import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
object StreamingConversions {
-
+
implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
new DataStream[R](javaStream)
@@ -33,4 +34,7 @@ object StreamingConversions {
implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
new SplitDataStream[R](javaStream)
+ implicit def javaToScalaConnectedStream[IN1, IN2](javaStream: JavaConStream[IN1, IN2]):
+ ConnectedDataStream[IN1, IN2] = new ConnectedDataStream[IN1, IN2](javaStream)
+
}
[31/36] flink git commit: [streaming] Deleted obsolete parts of the
connected stream api
Posted by mb...@apache.org.
[streaming] Deleted obsolete parts of the connected stream api
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fcd28fcc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fcd28fcc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fcd28fcc
Branch: refs/heads/release-0.8
Commit: fcd28fccd38a070f69676f48b2038af9c6e5eb4e
Parents: fb86fde
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Jan 5 15:59:50 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 00:22:58 2015 +0100
----------------------------------------------------------------------
.../api/datastream/CoBatchedDataStream.java | 126 --------
.../api/datastream/CoWindowDataStream.java | 112 -------
.../api/datastream/ConnectedDataStream.java | 167 ----------
.../operator/co/CoBatchReduceInvokable.java | 316 -------------------
.../co/CoGroupedBatchReduceInvokable.java | 79 -----
.../co/CoGroupedWindowReduceInvokable.java | 158 ----------
.../operator/co/CoWindowReduceInvokable.java | 189 -----------
.../invokable/operator/CoBatchReduceTest.java | 137 --------
.../operator/CoGroupedBatchReduceTest.java | 172 ----------
.../operator/CoGroupedWindowReduceTest.java | 211 -------------
.../invokable/operator/CoWindowReduceTest.java | 172 ----------
11 files changed, 1839 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoBatchedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoBatchedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoBatchedDataStream.java
deleted file mode 100644
index 3b58188..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoBatchedDataStream.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoBatchReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedBatchReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-
-/**
- * A {@link CoBatchedDataStream} represents a two data stream whose elements are
- * batched together in sliding batches. Operation
- * {@link #reduce(ReduceFunction)} can be applied for each batch and the batch
- * is slid afterwards.
- *
- * @param <IN1>
- * The type of the first input data stream
- * @param <IN2>
- * The type of the second input data stream
- */
-public class CoBatchedDataStream<IN1, IN2> extends ConnectedDataStream<IN1, IN2> {
-
- protected long batchSize1;
- protected long batchSize2;
- protected long slideSize1;
- protected long slideSize2;
-
- protected CoBatchedDataStream(DataStream<IN1> dataStream1, DataStream<IN2> dataStream2,
- long batchSize1, long batchSize2, long slideSize1, long slideSize2) {
- super(dataStream1, dataStream2);
- this.batchSize1 = batchSize1;
- this.batchSize2 = batchSize2;
- this.slideSize1 = slideSize1;
- this.slideSize2 = slideSize2;
- }
-
- protected CoBatchedDataStream(ConnectedDataStream<IN1, IN2> coDataStream, long batchSize1,
- long batchSize2, long slideSize1, long slideSize2) {
- super(coDataStream);
- this.batchSize1 = batchSize1;
- this.batchSize2 = batchSize2;
- this.slideSize1 = slideSize1;
- this.slideSize2 = slideSize2;
- }
-
- protected CoBatchedDataStream(CoBatchedDataStream<IN1, IN2> coBatchedDataStream) {
- super(coBatchedDataStream);
- this.batchSize1 = coBatchedDataStream.batchSize1;
- this.batchSize2 = coBatchedDataStream.batchSize2;
- this.slideSize1 = coBatchedDataStream.slideSize1;
- this.slideSize2 = coBatchedDataStream.slideSize2;
- }
-
- /**
- * Groups the elements of the {@link CoBatchedDataStream} by the given key
- * positions to be used with grouped operators.
- *
- * @param keyPosition1
- * The position of the field on which the first input data stream
- * will be grouped.
- * @param keyPosition2
- * The position of the field on which the second input data
- * stream will be grouped.
- * @return The transformed {@link CoBatchedDataStream}
- */
- public CoBatchedDataStream<IN1, IN2> groupBy(int keyPosition1, int keyPosition2) {
- return new CoBatchedDataStream<IN1, IN2>(dataStream1.groupBy(keyPosition1),
- dataStream2.groupBy(keyPosition2), batchSize1, batchSize2, slideSize1, slideSize2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(int[] keyPositions1, int[] keyPositions2) {
- return new CoBatchedDataStream<IN1, IN2>(dataStream1.groupBy(keyPositions1),
- dataStream2.groupBy(keyPositions2), batchSize1, batchSize2, slideSize1, slideSize2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(String field1, String field2) {
- return new CoBatchedDataStream<IN1, IN2>(dataStream1.groupBy(field1),
- dataStream2.groupBy(field2), batchSize1, batchSize2, slideSize1, slideSize2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(String[] fields1, String[] fields2) {
- return new CoBatchedDataStream<IN1, IN2>(dataStream1.groupBy(fields1),
- dataStream2.groupBy(fields2), batchSize1, batchSize2, slideSize1, slideSize2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(KeySelector<IN1, ?> keySelector1,
- KeySelector<IN2, ?> keySelector2) {
- return new CoBatchedDataStream<IN1, IN2>(dataStream1.groupBy(keySelector1),
- dataStream2.groupBy(keySelector2), batchSize1, batchSize2, slideSize1, slideSize2);
- }
-
- @Override
- protected <OUT> CoInvokable<IN1, IN2, OUT> getReduceInvokable(
- CoReduceFunction<IN1, IN2, OUT> coReducer) {
- CoBatchReduceInvokable<IN1, IN2, OUT> invokable;
- if (isGrouped) {
- invokable = new CoGroupedBatchReduceInvokable<IN1, IN2, OUT>(clean(coReducer), batchSize1,
- batchSize2, slideSize1, slideSize2, keySelector1, keySelector2);
- } else {
- invokable = new CoBatchReduceInvokable<IN1, IN2, OUT>(clean(coReducer), batchSize1,
- batchSize2, slideSize1, slideSize2);
- }
- return invokable;
- }
-
- protected CoBatchedDataStream<IN1, IN2> copy() {
- return new CoBatchedDataStream<IN1, IN2>(this);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java
deleted file mode 100644
index 9129f9e..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/CoWindowDataStream.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedWindowReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoInvokable;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowReduceInvokable;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-/**
- * A {@link CoWindowDataStream} represents two data streams whose elements are
- * batched together into sliding windows. Operation
- * {@link #reduce(CoReduceFunction)} can be applied for each window.
- *
- * @param <IN1>
- * The type of the first input data stream
- * @param <IN2>
- * The type of the second input data stream
- */
-public class CoWindowDataStream<IN1, IN2> extends CoBatchedDataStream<IN1, IN2> {
- TimestampWrapper<IN1> timeStamp1;
- TimestampWrapper<IN2> timeStamp2;
-
- protected CoWindowDataStream(DataStream<IN1> dataStream1, DataStream<IN2> dataStream2,
- long windowSize1, long windowSize2, long slideInterval1, long slideInterval2,
- TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
- super(dataStream1, dataStream2, windowSize1, windowSize2, slideInterval1, slideInterval2);
- this.timeStamp1 = timeStamp1;
- this.timeStamp2 = timeStamp2;
- }
-
- protected CoWindowDataStream(ConnectedDataStream<IN1, IN2> coDataStream, long windowSize1,
- long windowSize2, long slideInterval1, long slideInterval2,
- TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
- super(coDataStream, windowSize1, windowSize2, slideInterval1, slideInterval2);
- this.timeStamp1 = timeStamp1;
- this.timeStamp2 = timeStamp2;
- }
-
- protected CoWindowDataStream(CoWindowDataStream<IN1, IN2> coWindowDataStream) {
- super(coWindowDataStream);
- this.timeStamp1 = coWindowDataStream.timeStamp1;
- this.timeStamp2 = coWindowDataStream.timeStamp2;
- }
-
- public CoWindowDataStream<IN1, IN2> groupBy(int keyPosition1, int keyPosition2) {
- return new CoWindowDataStream<IN1, IN2>(dataStream1.groupBy(keyPosition1),
- dataStream2.groupBy(keyPosition2), batchSize1, batchSize2, slideSize1, slideSize2,
- timeStamp1, timeStamp2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(int[] keyPositions1, int[] keyPositions2) {
- return new CoWindowDataStream<IN1, IN2>(dataStream1.groupBy(keyPositions1),
- dataStream2.groupBy(keyPositions2), batchSize1, batchSize2, slideSize1, slideSize2,
- timeStamp1, timeStamp2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(String field1, String field2) {
- return new CoWindowDataStream<IN1, IN2>(dataStream1.groupBy(field1),
- dataStream2.groupBy(field2), batchSize1, batchSize2, slideSize1, slideSize2,
- timeStamp1, timeStamp2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(String[] fields1, String[] fields2) {
- return new CoWindowDataStream<IN1, IN2>(dataStream1.groupBy(fields1),
- dataStream2.groupBy(fields2), batchSize1, batchSize2, slideSize1, slideSize2,
- timeStamp1, timeStamp2);
- }
-
- public ConnectedDataStream<IN1, IN2> groupBy(KeySelector<IN1, ?> keySelector1,
- KeySelector<IN2, ?> keySelector2) {
- return new CoWindowDataStream<IN1, IN2>(dataStream1.groupBy(keySelector1),
- dataStream2.groupBy(keySelector2), batchSize1, batchSize2, slideSize1, slideSize2,
- timeStamp1, timeStamp2);
- }
-
- @Override
- protected <OUT> CoInvokable<IN1, IN2, OUT> getReduceInvokable(
- CoReduceFunction<IN1, IN2, OUT> coReducer) {
- CoWindowReduceInvokable<IN1, IN2, OUT> invokable;
- if (isGrouped) {
- invokable = new CoGroupedWindowReduceInvokable<IN1, IN2, OUT>(clean(coReducer),
- batchSize1, batchSize2, slideSize1, slideSize2, keySelector1, keySelector2,
- timeStamp1, timeStamp2);
- } else {
- invokable = new CoWindowReduceInvokable<IN1, IN2, OUT>(clean(coReducer), batchSize1,
- batchSize2, slideSize1, slideSize2, timeStamp1, timeStamp2);
- }
- return invokable;
- }
-
- protected CoWindowDataStream<IN1, IN2> copy() {
- return new CoWindowDataStream<IN1, IN2>(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index efd9531..a0c8ff8 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -227,173 +227,6 @@ public class ConnectedDataStream<IN1, IN2> {
}
/**
- * Batch operation for connected data stream. Collects each input data
- * stream's elements simultaneously into sliding batches creating a new
- * {@link CoBatchedDataStream}. Then the user can apply
- * {@link CoBatchedDataStream#reduce} transformation on the
- * {@link CoBatchedDataStream}.
- *
- * @param batchSize1
- * The number of elements in each batch of the first input data
- * stream
- * @param batchSize2
- * The number of elements in each batch of the second input data
- * stream
- * @param slideSize1
- * The number of elements with which the batches of the first
- * input data stream are slid by after each transformation.
- * @param slideSize2
- * The number of elements with which the batches of the second
- * input data stream are slid by after each transformation.
- * @return The transformed {@link ConnectedDataStream}
- */
- public CoBatchedDataStream<IN1, IN2> batch(long batchSize1, long batchSize2, long slideSize1,
- long slideSize2) {
- if (batchSize1 < 1 || batchSize2 < 1) {
- throw new IllegalArgumentException("Batch size must be positive");
- }
- if (slideSize1 < 1 || slideSize2 < 1) {
- throw new IllegalArgumentException("Slide size must be positive");
- }
- return new CoBatchedDataStream<IN1, IN2>(this, batchSize1, batchSize2, slideSize1,
- slideSize2);
- }
-
- /**
- * Batch operation for connected data stream. Collects each input data
- * stream's elements simultaneously into batches creating a new
- * {@link CoBatchedDataStream}. Then the user can apply
- * {@link CoBatchedDataStream#reduce} transformation on the
- * {@link CoBatchedDataStream}.
- *
- * @param batchSize1
- * The number of elements in each batch of the first input data
- * stream
- * @param batchSize2
- * The number of elements in each batch of the second input data
- * stream
- * @return The transformed {@link ConnectedDataStream}
- */
- public CoBatchedDataStream<IN1, IN2> batch(long batchSize1, long batchSize2) {
- return batch(batchSize1, batchSize2, batchSize1, batchSize2);
- }
-
- /**
- * Window operation for connected data stream. Collects each input data
- * stream's elements simultaneously into sliding windows creating a new
- * {@link CoWindowDataStream}. Then the user can apply
- * {@link WindowDataStream#reduce} transformation on the
- * {@link CoWindowDataStream}. The user can implement their own time stamps
- * or use the system time by default.
- *
- * @param windowSize1
- * The length of the window of the first input data stream
- * @param windowSize2
- * The length of the window of the second input data stream
- * @param slideInterval1
- * The number of milliseconds with which the windows of the first
- * input data stream are slid by after each transformation
- * @param slideInterval2
- * The number of milliseconds with which the windows of the
- * second input data stream are slid by after each transformation
- * @param timeStamp1
- * User defined function for extracting time-stamps from each
- * element of the first input data stream
- * @param timeStamp2
- * User defined function for extracting time-stamps from each
- * element of the second input data stream
- * @return The transformed {@link ConnectedDataStream}
- */
- public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2,
- long slideInterval1, long slideInterval2, TimestampWrapper<IN1> timeStamp1,
- TimestampWrapper<IN2> timeStamp2) {
- if (windowSize1 < 1 || windowSize2 < 1) {
- throw new IllegalArgumentException("Window size must be positive");
- }
- if (slideInterval1 < 1 || slideInterval2 < 1) {
- throw new IllegalArgumentException("Slide interval must be positive");
- }
- return new CoWindowDataStream<IN1, IN2>(this, windowSize1, windowSize2, slideInterval1,
- slideInterval2, timeStamp1, timeStamp2);
- }
-
- /**
- * Window operation for connected data stream. Collects each input data
- * stream's elements simultaneously into sliding windows creating a new
- * {@link CoWindowDataStream}. Then the user can apply
- * {@link WindowDataStream#reduce} transformation on the
- * {@link CoWindowDataStream}.
- *
- * @param windowSize1
- * The length of the window of the first input data stream in
- * milliseconds.
- * @param windowSize2
- * The length of the window of the second input data stream in
- * milliseconds.
- * @param slideInterval1
- * The number of milliseconds with which the windows of the first
- * input data stream are slid by after each transformation
- * @param slideInterval2
- * The number of milliseconds with which the windows of the
- * second input data stream are slid by after each transformation
- * @return The transformed {@link ConnectedDataStream}
- */
- @SuppressWarnings("unchecked")
- public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2,
- long slideInterval1, long slideInterval2) {
- return window(windowSize1, windowSize2, slideInterval1, slideInterval2,
- (TimestampWrapper<IN1>) SystemTimestamp.getWrapper(),
- (TimestampWrapper<IN2>) SystemTimestamp.getWrapper());
- }
-
- /**
- * Window operation for connected data stream. Collects each input data
- * stream's elements simultaneously into windows creating a new
- * {@link CoWindowDataStream}. Then the user can apply
- * {@link WindowDataStream#reduce} transformation on the
- * {@link CoWindowDataStream}. The user can implement their own time stamps
- * or use the system time by default.
- *
- * @param windowSize1
- * The length of the window of the first input data stream
- * @param windowSize2
- * The length of the window of the second input data stream
- * @param timeStamp1
- * User defined function for extracting time-stamps from each
- * element of the first input data stream
- * @param timeStamp2
- * User defined function for extracting time-stamps from each
- * element of the second input data stream
- * @return The transformed {@link ConnectedDataStream}
- */
- public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2,
- TimestampWrapper<IN1> timeStamp1, TimestampWrapper<IN2> timeStamp2) {
- return window(windowSize1, windowSize2, windowSize1, windowSize2, timeStamp1, timeStamp2);
- }
-
- /**
- * Window operation for connected data stream. Collects each input data
- * stream's elements simultaneously into windows creating a new
- * {@link CoWindowDataStream}. Then the user can apply
- * {@link WindowDataStream#reduce} transformation on the
- * {@link CoWindowDataStream}.
- *
- * @param windowSize1
- * The length of the window of the first input data stream in
- * milliseconds
- * @param windowSize2
- * The length of the window of the second input data stream in
- * milliseconds
- * @return The transformed {@link ConnectedDataStream}
- */
- @SuppressWarnings("unchecked")
- public CoWindowDataStream<IN1, IN2> window(long windowSize1, long windowSize2) {
- return window(windowSize1, windowSize2, windowSize1, windowSize2,
- (TimestampWrapper<IN1>) SystemTimestamp.getWrapper(),
- (TimestampWrapper<IN2>) SystemTimestamp.getWrapper());
- }
-
- /**
* Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
* the output to a common type. The transformation calls a
* {@link CoMapFunction#map1} for each element of the first input and
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoBatchReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoBatchReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoBatchReduceInvokable.java
deleted file mode 100644
index 4ed49fd..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoBatchReduceInvokable.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-import org.apache.commons.math.util.MathUtils;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.state.NullableCircularBuffer;
-
-public class CoBatchReduceInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT> {
-
- private static final long serialVersionUID = 1L;
- protected CoReduceFunction<IN1, IN2, OUT> coReducer;
-
- protected long slideSize1;
- protected long slideSize2;
- protected long batchSize1;
- protected long batchSize2;
- protected int granularity1;
- protected int granularity2;
- protected long batchPerSlide1;
- protected long batchPerSlide2;
- protected long numberOfBatches1;
- protected long numberOfBatches2;
- protected StreamBatch<IN1> batch1;
- protected StreamBatch<IN2> batch2;
- protected StreamBatch<IN1> currentBatch1;
- protected StreamBatch<IN2> currentBatch2;
-
- public CoBatchReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer, long batchSize1,
- long batchSize2, long slideSize1, long slideSize2) {
- super(coReducer);
- this.coReducer = coReducer;
- this.batchSize1 = batchSize1;
- this.batchSize2 = batchSize2;
- this.slideSize1 = slideSize1;
- this.slideSize2 = slideSize2;
- this.granularity1 = (int) MathUtils.gcd(batchSize1, slideSize1);
- this.granularity2 = (int) MathUtils.gcd(batchSize2, slideSize2);
- this.batchPerSlide1 = slideSize1 / granularity1;
- this.batchPerSlide2 = slideSize2 / granularity2;
- this.numberOfBatches1 = batchSize1 / granularity1;
- this.numberOfBatches2 = batchSize2 / granularity2;
- }
-
- @Override
- public void invoke() throws Exception {
- while (true) {
- int next = recordIterator.next(reuse1, reuse2);
- if (next == 0) {
- reduceLastBatch1();
- reduceLastBatch2();
- break;
- } else if (next == 1) {
- handleStream1();
- resetReuse1();
- } else {
- handleStream2();
- resetReuse2();
- }
- }
- }
-
- @Override
- protected void handleStream1() throws Exception {
- StreamBatch<IN1> batch1 = getBatch1(reuse1);
- reduceToBuffer1(reuse1.getObject(), batch1);
- }
-
- @Override
- protected void handleStream2() throws Exception {
- StreamBatch<IN2> batch2 = getBatch2(reuse2);
- reduceToBuffer2(reuse2.getObject(), batch2);
- }
-
- protected StreamBatch<IN1> getBatch1(StreamRecord<IN1> next) {
- return batch1;
- }
-
- protected StreamBatch<IN2> getBatch2(StreamRecord<IN2> next) {
- return batch2;
- }
-
- protected void reduce1(StreamBatch<IN1> batch) {
- this.currentBatch1 = batch;
- callUserFunctionAndLogException1();
- }
-
- protected void reduce2(StreamBatch<IN2> batch) {
- this.currentBatch2 = batch;
- callUserFunctionAndLogException2();
- }
-
- protected void reduceLastBatch1() throws Exception {
- reduceLastBatch1(batch1);
- }
-
- protected void reduceLastBatch2() throws Exception {
- reduceLastBatch2(batch2);
- }
-
- @Override
- protected void callUserFunction1() throws Exception {
- Iterator<IN1> reducedIterator = currentBatch1.getIterator();
- IN1 reduced = null;
-
- while (reducedIterator.hasNext() && reduced == null) {
- reduced = reducedIterator.next();
- }
-
- while (reducedIterator.hasNext()) {
- IN1 next = reducedIterator.next();
- if (next != null) {
- reduced = coReducer.reduce1(serializer1.copy(reduced), serializer1.copy(next));
- }
- }
- if (reduced != null) {
- collector.collect(coReducer.map1(serializer1.copy(reduced)));
- }
- }
-
- @Override
- protected void callUserFunction2() throws Exception {
- Iterator<IN2> reducedIterator = currentBatch2.getIterator();
- IN2 reduced = null;
-
- while (reducedIterator.hasNext() && reduced == null) {
- reduced = reducedIterator.next();
- }
-
- while (reducedIterator.hasNext()) {
- IN2 next = reducedIterator.next();
- if (next != null) {
- reduced = coReducer.reduce2(serializer2.copy(reduced), serializer2.copy(next));
- }
- }
- if (reduced != null) {
- collector.collect(coReducer.map2(serializer2.copy(reduced)));
- }
- }
-
- @Override
- public void open(Configuration config) throws Exception {
- super.open(config);
- this.batch1 = new StreamBatch<IN1>(batchSize1, slideSize1);
- this.batch2 = new StreamBatch<IN2>(batchSize2, slideSize2);
- }
-
- public void reduceToBuffer1(IN1 nextValue, StreamBatch<IN1> streamBatch) throws Exception {
-
- if (streamBatch.currentValue != null) {
- streamBatch.currentValue = coReducer.reduce1(
- serializer1.copy(streamBatch.currentValue), serializer1.copy(nextValue));
- } else {
- streamBatch.currentValue = nextValue;
- }
-
- streamBatch.counter++;
-
- if (streamBatch.miniBatchEnd()) {
- streamBatch.addToBuffer();
- if (streamBatch.batchEnd()) {
- reduceBatch1(streamBatch);
- }
- }
- }
-
- public void reduceToBuffer2(IN2 nextValue, StreamBatch<IN2> streamBatch) throws Exception {
-
- if (streamBatch.currentValue != null) {
- streamBatch.currentValue = coReducer.reduce2(
- serializer2.copy(streamBatch.currentValue), serializer2.copy(nextValue));
- } else {
- streamBatch.currentValue = nextValue;
- }
-
- streamBatch.counter++;
-
- if (streamBatch.miniBatchEnd()) {
- streamBatch.addToBuffer();
- if (streamBatch.batchEnd()) {
- reduceBatch2(streamBatch);
- }
- }
- }
-
- public void reduceLastBatch1(StreamBatch<IN1> streamBatch) throws Exception {
- if (streamBatch.miniBatchInProgress()) {
- streamBatch.addToBuffer();
- }
-
- if (streamBatch.changed == true && streamBatch.minibatchCounter >= 0) {
- if (streamBatch.circularBuffer.isFull()) {
- for (long i = 0; i < (numberOfBatches1 - streamBatch.minibatchCounter); i++) {
- if (!streamBatch.circularBuffer.isEmpty()) {
- streamBatch.circularBuffer.remove();
- }
- }
- }
- if (!streamBatch.circularBuffer.isEmpty()) {
- reduce1(streamBatch);
- }
- }
-
- }
-
- public void reduceLastBatch2(StreamBatch<IN2> streamBatch) throws Exception {
- if (streamBatch.miniBatchInProgress()) {
- streamBatch.addToBuffer();
- }
-
- if (streamBatch.changed == true && streamBatch.minibatchCounter >= 0) {
- if (streamBatch.circularBuffer.isFull()) {
- for (long i = 0; i < (numberOfBatches2 - streamBatch.minibatchCounter); i++) {
- if (!streamBatch.circularBuffer.isEmpty()) {
- streamBatch.circularBuffer.remove();
- }
- }
- }
- if (!streamBatch.circularBuffer.isEmpty()) {
- reduce2(streamBatch);
- }
- }
-
- }
-
- public void reduceBatch1(StreamBatch<IN1> streamBatch) {
- reduce1(streamBatch);
- streamBatch.changed = false;
- }
-
- public void reduceBatch2(StreamBatch<IN2> streamBatch) {
- reduce2(streamBatch);
- streamBatch.changed = false;
- }
-
- protected class StreamBatch<IN> implements Serializable {
- private static final long serialVersionUID = 1L;
-
- protected long counter;
- protected long minibatchCounter;
- protected IN currentValue;
- protected long batchSize;
- protected long slideSize;
- protected long granularity;
- protected long batchPerSlide;
- protected long numberOfBatches;
- boolean changed;
-
- protected NullableCircularBuffer circularBuffer;
-
- public StreamBatch(long batchSize, long slideSize) {
- this.batchSize = batchSize;
- this.slideSize = slideSize;
- this.granularity = (int) MathUtils.gcd(batchSize, slideSize);
- this.batchPerSlide = slideSize / granularity;
- this.circularBuffer = new NullableCircularBuffer((int) (batchSize / granularity));
- this.counter = 0;
- this.minibatchCounter = 0;
- this.currentValue = null;
- this.numberOfBatches = batchSize / granularity;
- this.changed = false;
-
- }
-
- protected void addToBuffer() {
- circularBuffer.add(currentValue);
- changed = true;
- minibatchCounter++;
- currentValue = null;
- }
-
- protected boolean miniBatchEnd() {
- return (counter % granularity) == 0;
- }
-
- public boolean batchEnd() {
- if (counter == batchSize) {
- counter -= slideSize;
- minibatchCounter -= batchPerSlide;
- return true;
- }
- return false;
- }
-
- public boolean miniBatchInProgress() {
- return currentValue != null;
- }
-
- @SuppressWarnings("unchecked")
- public Iterator<IN> getIterator() {
- return circularBuffer.iterator();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedBatchReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedBatchReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedBatchReduceInvokable.java
deleted file mode 100644
index 745f507..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedBatchReduceInvokable.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-public class CoGroupedBatchReduceInvokable<IN1, IN2, OUT> extends
- CoBatchReduceInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
- protected KeySelector<IN1, ?> keySelector1;
- protected KeySelector<IN2, ?> keySelector2;
- Map<Object, StreamBatch<IN1>> streamBatches1;
- Map<Object, StreamBatch<IN2>> streamBatches2;
-
- public CoGroupedBatchReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer,
- long batchSize1, long batchSize2, long slideSize1, long slideSize2,
- KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2) {
- super(coReducer, batchSize1, batchSize2, slideSize1, slideSize2);
- this.keySelector1 = keySelector1;
- this.keySelector2 = keySelector2;
- this.streamBatches1 = new HashMap<Object, StreamBatch<IN1>>();
- this.streamBatches2 = new HashMap<Object, StreamBatch<IN2>>();
- }
-
- protected void reduceLastBatch1() throws Exception {
- for (StreamBatch<IN1> batch : streamBatches1.values()) {
- reduceLastBatch1(batch);
- }
- }
-
- protected void reduceLastBatch2() throws Exception {
- for (StreamBatch<IN2> batch : streamBatches2.values()) {
- reduceLastBatch2(batch);
- }
- }
-
- @Override
- protected StreamBatch<IN1> getBatch1(StreamRecord<IN1> next) {
- Object key = next.getKey(keySelector1);
- StreamBatch<IN1> batch = streamBatches1.get(key);
- if (batch == null) {
- batch = new StreamBatch<IN1>(batchSize1, slideSize1);
- streamBatches1.put(key, batch);
- }
- return batch;
- }
-
- @Override
- protected StreamBatch<IN2> getBatch2(StreamRecord<IN2> next) {
- Object key = next.getKey(keySelector2);
- StreamBatch<IN2> batch = streamBatches2.get(key);
- if (batch == null) {
- batch = new StreamBatch<IN2>(batchSize2, slideSize2);
- streamBatches2.put(key, batch);
- }
- return batch;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java
deleted file mode 100644
index 736239f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedWindowReduceInvokable.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-public class CoGroupedWindowReduceInvokable<IN1, IN2, OUT> extends
- CoWindowReduceInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
- protected KeySelector<IN1, ?> keySelector1;
- protected KeySelector<IN2, ?> keySelector2;
- private Map<Object, StreamWindow<IN1>> streamWindows1;
- private Map<Object, StreamWindow<IN2>> streamWindows2;
- private long currentMiniBatchCount1 = 0;
- private long currentMiniBatchCount2 = 0;
-
- public CoGroupedWindowReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer,
- long windowSize1, long windowSize2, long slideInterval1, long slideInterval2,
- KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2,
- TimestampWrapper<IN1> timestamp1, TimestampWrapper<IN2> timestamp2) {
- super(coReducer, windowSize1, windowSize2, slideInterval1, slideInterval2, timestamp1,
- timestamp2);
- this.keySelector1 = keySelector1;
- this.keySelector2 = keySelector2;
- this.streamWindows1 = new HashMap<Object, StreamWindow<IN1>>();
- this.streamWindows2 = new HashMap<Object, StreamWindow<IN2>>();
- }
-
- @Override
- protected StreamBatch<IN1> getBatch1(StreamRecord<IN1> next) {
- Object key = next.getKey(keySelector1);
- StreamWindow<IN1> window = streamWindows1.get(key);
- if (window == null) {
- window = new GroupedStreamWindow<IN1>(batchSize1, slideSize1);
- window.minibatchCounter = currentMiniBatchCount1;
- streamWindows1.put(key, window);
- }
- this.window1 = window;
- return window;
- }
-
- @Override
- protected StreamBatch<IN2> getBatch2(StreamRecord<IN2> next) {
- Object key = next.getKey(keySelector2);
- StreamWindow<IN2> window = streamWindows2.get(key);
- if (window == null) {
- window = new GroupedStreamWindow<IN2>(batchSize2, slideSize2);
- window.minibatchCounter = currentMiniBatchCount2;
- streamWindows2.put(key, window);
- }
- this.window2 = window;
- return window;
- }
-
- private void addToAllBuffers1() {
- for (StreamBatch<IN1> window : streamWindows1.values()) {
- window.addToBuffer();
- }
- }
-
- private void addToAllBuffers2() {
- for (StreamBatch<IN2> window : streamWindows2.values()) {
- window.addToBuffer();
- }
- }
-
- private void reduceAllWindows1() {
- for (StreamBatch<IN1> window : streamWindows1.values()) {
- window.minibatchCounter -= batchPerSlide1;
- reduceBatch1((StreamBatch<IN1>) window);
- }
- }
-
- private void reduceAllWindows2() {
- for (StreamBatch<IN2> window : streamWindows2.values()) {
- window.minibatchCounter -= batchPerSlide2;
- reduceBatch2((StreamBatch<IN2>) window);
- }
- }
-
- @Override
- protected void reduceLastBatch1() throws Exception {
- for (StreamBatch<IN1> window : streamWindows1.values()) {
- reduceLastBatch1((StreamBatch<IN1>) window);
- }
- }
-
- @Override
- protected void reduceLastBatch2() throws Exception {
- for (StreamBatch<IN2> window : streamWindows2.values()) {
- reduceLastBatch2((StreamBatch<IN2>) window);
- }
- }
-
- @Override
- protected synchronized void checkWindowEnd1(long timeStamp, StreamWindow<IN1> streamWindow) {
- nextRecordTime1 = timeStamp;
-
- while (miniBatchEnd1()) {
- addToAllBuffers1();
- if (streamWindow.batchEnd()) {
- reduceAllWindows1();
- }
- }
- currentMiniBatchCount1 = streamWindow.minibatchCounter;
- }
-
- @Override
- protected synchronized void checkWindowEnd2(long timeStamp, StreamWindow<IN2> streamWindow) {
- nextRecordTime2 = timeStamp;
-
- while (miniBatchEnd2()) {
- addToAllBuffers2();
- if (streamWindow.batchEnd()) {
- reduceAllWindows2();
- }
- }
- currentMiniBatchCount2 = streamWindow.minibatchCounter;
- }
-
- protected class GroupedStreamWindow<IN> extends StreamWindow<IN> {
- private static final long serialVersionUID = 1L;
-
- public GroupedStreamWindow(long windowSize, long slideInterval) {
- super(windowSize, slideInterval);
- }
-
- @Override
- public boolean batchEnd() {
- if (minibatchCounter == numberOfBatches) {
- return true;
- }
- return false;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java
deleted file mode 100644
index 0c8598f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowReduceInvokable.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.co;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-public class CoWindowReduceInvokable<IN1, IN2, OUT> extends CoBatchReduceInvokable<IN1, IN2, OUT> {
- private static final long serialVersionUID = 1L;
- protected long startTime1;
- protected long startTime2;
- protected long nextRecordTime1;
- protected long nextRecordTime2;
- protected TimestampWrapper<IN1> timestamp1;
- protected TimestampWrapper<IN2> timestamp2;
- protected StreamWindow<IN1> window1;
- protected StreamWindow<IN2> window2;
-
- public CoWindowReduceInvokable(CoReduceFunction<IN1, IN2, OUT> coReducer, long windowSize1,
- long windowSize2, long slideInterval1, long slideInterval2,
- TimestampWrapper<IN1> timestamp1, TimestampWrapper<IN2> timestamp2) {
- super(coReducer, windowSize1, windowSize2, slideInterval1, slideInterval2);
- this.timestamp1 = timestamp1;
- this.timestamp2 = timestamp2;
- this.startTime1 = timestamp1.getStartTime();
- this.startTime2 = timestamp2.getStartTime();
-
- }
-
- @Override
- public void open(Configuration config) throws Exception {
- super.open(config);
- this.window1 = new StreamWindow<IN1>(batchSize1, slideSize1);
- this.window2 = new StreamWindow<IN2>(batchSize2, slideSize2);
- this.batch1 = this.window1;
- this.batch2 = this.window2;
- if (timestamp1.isDefaultTimestamp()) {
- (new TimeCheck1()).start();
- }
- if (timestamp2.isDefaultTimestamp()) {
- (new TimeCheck2()).start();
- }
- }
-
- @Override
- public void reduceToBuffer1(IN1 nextValue, StreamBatch<IN1> streamWindow) throws Exception {
-
- checkWindowEnd1(timestamp1.getTimestamp(nextValue), (StreamWindow<IN1>) streamWindow);
-
- if (streamWindow.currentValue != null) {
- streamWindow.currentValue = coReducer.reduce1(
- serializer1.copy(streamWindow.currentValue), serializer1.copy(nextValue));
- } else {
- streamWindow.currentValue = nextValue;
- }
- }
-
- @Override
- public void reduceToBuffer2(IN2 nextValue, StreamBatch<IN2> streamWindow) throws Exception {
-
- checkWindowEnd2(timestamp2.getTimestamp(nextValue), (StreamWindow<IN2>) streamWindow);
-
- if (streamWindow.currentValue != null) {
- streamWindow.currentValue = coReducer.reduce2(
- serializer2.copy(streamWindow.currentValue), serializer2.copy(nextValue));
- } else {
- streamWindow.currentValue = nextValue;
- }
- }
-
- protected synchronized void checkWindowEnd1(long timeStamp, StreamWindow<IN1> streamWindow) {
- nextRecordTime1 = timeStamp;
-
- while (miniBatchEnd1()) {
- streamWindow.addToBuffer();
- if (streamWindow.batchEnd()) {
- reduceBatch1(streamWindow);
- }
- }
- }
-
- protected synchronized void checkWindowEnd2(long timeStamp, StreamWindow<IN2> streamWindow) {
- nextRecordTime2 = timeStamp;
-
- while (miniBatchEnd2()) {
- streamWindow.addToBuffer();
- if (streamWindow.batchEnd()) {
- reduceBatch2(streamWindow);
- }
- }
- }
-
- protected boolean miniBatchEnd1() {
- if (nextRecordTime1 < startTime1 + granularity1) {
- return false;
- } else {
- startTime1 += granularity1;
- return true;
- }
- }
-
- protected boolean miniBatchEnd2() {
- if (nextRecordTime2 < startTime2 + granularity2) {
- return false;
- } else {
- startTime2 += granularity2;
- return true;
- }
- }
-
- @Override
- public void reduceBatch1(StreamBatch<IN1> streamBatch) {
- reduce1(streamBatch);
- }
-
- @Override
- public void reduceBatch2(StreamBatch<IN2> streamBatch) {
- reduce2(streamBatch);
- }
-
- protected class StreamWindow<IN> extends StreamBatch<IN> {
- private static final long serialVersionUID = 1L;
-
- public StreamWindow(long windowSize, long slideInterval) {
- super(windowSize, slideInterval);
- }
-
- @Override
- public boolean batchEnd() {
- if (minibatchCounter == numberOfBatches) {
- minibatchCounter -= batchPerSlide;
- return true;
- }
- return false;
- }
-
- }
-
- private class TimeCheck1 extends Thread {
- @Override
- public void run() {
- while (true) {
- try {
- Thread.sleep(slideSize1);
- } catch (InterruptedException e) {
- }
- if (isRunning) {
- checkWindowEnd1(System.currentTimeMillis(), window1);
- } else {
- break;
- }
- }
- }
- }
-
- private class TimeCheck2 extends Thread {
- @Override
- public void run() {
- while (true) {
- try {
- Thread.sleep(slideSize2);
- } catch (InterruptedException e) {
- }
- if (isRunning) {
- checkWindowEnd2(System.currentTimeMillis(), window2);
- } else {
- break;
- }
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoBatchReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoBatchReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoBatchReduceTest.java
deleted file mode 100644
index 1db286c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoBatchReduceTest.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoBatchReduceInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoBatchReduceTest {
-
- private static class MyCoReduceFunction implements CoReduceFunction<Integer, String, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce1(Integer value1, Integer value2) {
- return value1 + value2;
- }
-
- @Override
- public String reduce2(String value1, String value2) {
- return value1 + value2;
- }
-
- @Override
- public String map1(Integer value) {
- return value.toString();
- }
-
- @Override
- public String map2(String value) {
- return value;
- }
- }
-
- @Test
- public void coBatchReduceTest1() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
-
- List<String> inputs2 = new ArrayList<String>();
- inputs2.add("a");
- inputs2.add("b");
- inputs2.add("c");
- inputs2.add("d");
- inputs2.add("e");
- inputs2.add("f");
- inputs2.add("g");
- inputs2.add("h");
- inputs2.add("i");
-
- CoBatchReduceInvokable<Integer, String, String> invokable = new CoBatchReduceInvokable<Integer, String, String>(
- new MyCoReduceFunction(), 4L, 3L, 4L, 3L);
-
- List<String> expected = new ArrayList<String>();
- expected.add("10");
- expected.add("26");
- expected.add("19");
- expected.add("abc");
- expected.add("def");
- expected.add("ghi");
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
-
- assertEquals(expected, result);
-
- }
-
- @Test
- public void coBatchReduceTest2() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
-
- List<String> inputs2 = new ArrayList<String>();
- inputs2.add("a");
- inputs2.add("b");
- inputs2.add("c");
- inputs2.add("d");
- inputs2.add("e");
- inputs2.add("f");
- inputs2.add("g");
- inputs2.add("h");
- inputs2.add("i");
-
- CoBatchReduceInvokable<Integer, String, String> invokable = new CoBatchReduceInvokable<Integer, String, String>(
- new MyCoReduceFunction(), 4L, 3L, 2L, 2L);
-
- List<String> expected = new ArrayList<String>();
- expected.add("10");
- expected.add("18");
- expected.add("26");
- expected.add("34");
- expected.add("abc");
- expected.add("cde");
- expected.add("efg");
- expected.add("ghi");
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
-
- assertEquals(expected, result);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java
deleted file mode 100644
index 1d0732c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedBatchReduceTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedBatchReduceInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoGroupedBatchReduceTest {
-
- KeySelector<Tuple2<String, String>, ?> keySelector1 = new KeySelector<Tuple2<String, String>, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple2<String, String> value) throws Exception {
- return value.f0;
- }
- };
-
- KeySelector<Tuple2<String, Integer>, ?> keySelector2 = new KeySelector<Tuple2<String, Integer>, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple2<String, Integer> value) throws Exception {
- return value.f0;
- }
- };
-
- private static class MyCoReduceFunction implements
- CoReduceFunction<Tuple2<String, Integer>, Tuple2<String, String>, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, Integer> reduce1(Tuple2<String, Integer> value1,
- Tuple2<String, Integer> value2) {
- return new Tuple2<String, Integer>("a", value1.f1 + value2.f1);
- }
-
- @Override
- public Tuple2<String, String> reduce2(Tuple2<String, String> value1,
- Tuple2<String, String> value2) {
- return new Tuple2<String, String>("a", value1.f1 + value2.f1);
- }
-
- @Override
- public String map1(Tuple2<String, Integer> value) {
- return value.f1.toString();
- }
-
- @Override
- public String map2(Tuple2<String, String> value) {
- return value.f1;
- }
- }
-
- @Test
- public void coGroupedBatchReduceTest1() {
-
- List<Tuple2<String, Integer>> inputs1 = new ArrayList<Tuple2<String, Integer>>();
- inputs1.add(new Tuple2<String, Integer>("a", 1));
- inputs1.add(new Tuple2<String, Integer>("a", 2));
- inputs1.add(new Tuple2<String, Integer>("a", 3));
- inputs1.add(new Tuple2<String, Integer>("a", 4));
- inputs1.add(new Tuple2<String, Integer>("a", 5));
- inputs1.add(new Tuple2<String, Integer>("b", 6));
- inputs1.add(new Tuple2<String, Integer>("a", 7));
- inputs1.add(new Tuple2<String, Integer>("b", 8));
- inputs1.add(new Tuple2<String, Integer>("b", 9));
- inputs1.add(new Tuple2<String, Integer>("b", 10));
-
- List<Tuple2<String, String>> inputs2 = new ArrayList<Tuple2<String, String>>();
- inputs2.add(new Tuple2<String, String>("1", "a"));
- inputs2.add(new Tuple2<String, String>("2", "b"));
- inputs2.add(new Tuple2<String, String>("1", "c"));
- inputs2.add(new Tuple2<String, String>("2", "d"));
- inputs2.add(new Tuple2<String, String>("1", "e"));
- inputs2.add(new Tuple2<String, String>("2", "f"));
- inputs2.add(new Tuple2<String, String>("1", "g"));
- inputs2.add(new Tuple2<String, String>("2", "h"));
- inputs2.add(new Tuple2<String, String>("1", "i"));
-
- List<String> expected = new ArrayList<String>();
- expected.add("10");
- expected.add("12");
- expected.add("33");
- expected.add("ace");
- expected.add("gi");
- expected.add("bdf");
- expected.add("h");
-
- CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 4L, 3L, keySelector2, keySelector1);
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
- assertEquals(expected, result);
- }
-
- @Test
- public void coGroupedBatchReduceTest2() {
-
- List<Tuple2<String, Integer>> inputs1 = new ArrayList<Tuple2<String, Integer>>();
- inputs1.add(new Tuple2<String, Integer>("a", 1));
- inputs1.add(new Tuple2<String, Integer>("a", 2));
- inputs1.add(new Tuple2<String, Integer>("a", 3));
- inputs1.add(new Tuple2<String, Integer>("a", 4));
- inputs1.add(new Tuple2<String, Integer>("a", 5));
- inputs1.add(new Tuple2<String, Integer>("b", 6));
- inputs1.add(new Tuple2<String, Integer>("a", 7));
- inputs1.add(new Tuple2<String, Integer>("b", 8));
- inputs1.add(new Tuple2<String, Integer>("b", 9));
- inputs1.add(new Tuple2<String, Integer>("b", 10));
-
- List<Tuple2<String, String>> inputs2 = new ArrayList<Tuple2<String, String>>();
- inputs2.add(new Tuple2<String, String>("1", "a"));
- inputs2.add(new Tuple2<String, String>("2", "b"));
- inputs2.add(new Tuple2<String, String>("1", "c"));
- inputs2.add(new Tuple2<String, String>("2", "d"));
- inputs2.add(new Tuple2<String, String>("1", "e"));
- inputs2.add(new Tuple2<String, String>("2", "f"));
- inputs2.add(new Tuple2<String, String>("1", "g"));
- inputs2.add(new Tuple2<String, String>("2", "h"));
- inputs2.add(new Tuple2<String, String>("1", "i"));
-
- List<String> expected = new ArrayList<String>();
- expected.add("10");
- expected.add("19");
- expected.add("33");
- expected.add("ace");
- expected.add("egi");
- expected.add("bdf");
- expected.add("fh");
-
- CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedBatchReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 2L, 2L, keySelector2, keySelector1);
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
- assertEquals(expected, result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
deleted file mode 100644
index 508366c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedWindowReduceTest.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedWindowReduceInvokable;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoGroupedWindowReduceTest {
-
- KeySelector<Tuple2<String, Integer>, ?> keySelector0 = new KeySelector<Tuple2<String, Integer>, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple2<String, Integer> value) throws Exception {
- return value.f0;
- }
- };
-
- KeySelector<Tuple2<String, String>, ?> keySelector1 = new KeySelector<Tuple2<String, String>, String>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public String getKey(Tuple2<String, String> value) throws Exception {
- return value.f0;
- }
- };
-
- private static class MyCoReduceFunction implements
- CoReduceFunction<Tuple2<String, Integer>, Tuple2<String, String>, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, Integer> reduce1(Tuple2<String, Integer> value1,
- Tuple2<String, Integer> value2) {
- return new Tuple2<String, Integer>("a", value1.f1 + value2.f1);
- }
-
- @Override
- public Tuple2<String, String> reduce2(Tuple2<String, String> value1,
- Tuple2<String, String> value2) {
- return new Tuple2<String, String>("a", value1.f1 + value2.f1);
- }
-
- @Override
- public String map1(Tuple2<String, Integer> value) {
- return value.f1.toString();
- }
-
- @Override
- public String map2(Tuple2<String, String> value) {
- return value.f1;
- }
- }
-
- public static final class MyTimeStamp<T> extends TimestampWrapper<T> {
- private static final long serialVersionUID = 1L;
-
- private Iterator<Long> timestamps;
- private long start;
-
- public MyTimeStamp(List<Long> timestamps) {
- super(null, 0);
- this.timestamps = timestamps.iterator();
- this.start = timestamps.get(0);
- }
-
- @Override
- public long getTimestamp(T value) {
- long ts = timestamps.next();
- return ts;
- }
-
- @Override
- public long getStartTime() {
- return start;
- }
- }
-
- @Test
- public void coGroupedWindowReduceTest1() {
-
- List<Long> timestamps1 = Arrays.asList(0L, 0L, 1L, 1L, 1L, 1L, 2L, 4L, 5L, 6L);
- List<Tuple2<String, Integer>> inputs1 = new ArrayList<Tuple2<String, Integer>>();
- inputs1.add(new Tuple2<String, Integer>("a", 1));
- inputs1.add(new Tuple2<String, Integer>("a", 2));
- inputs1.add(new Tuple2<String, Integer>("a", 3));
- inputs1.add(new Tuple2<String, Integer>("a", 4));
- inputs1.add(new Tuple2<String, Integer>("a", 5));
- inputs1.add(new Tuple2<String, Integer>("b", 6));
- inputs1.add(new Tuple2<String, Integer>("a", 7));
- inputs1.add(new Tuple2<String, Integer>("b", 8));
- inputs1.add(new Tuple2<String, Integer>("b", 9));
- inputs1.add(new Tuple2<String, Integer>("b", 10));
-
- List<Long> timestamps2 = Arrays.asList(1L, 1L, 2L, 2L, 3L, 5L, 5L, 6L, 7L);
- List<Tuple2<String, String>> inputs2 = new ArrayList<Tuple2<String, String>>();
- inputs2.add(new Tuple2<String, String>("1", "a"));
- inputs2.add(new Tuple2<String, String>("2", "b"));
- inputs2.add(new Tuple2<String, String>("1", "c"));
- inputs2.add(new Tuple2<String, String>("2", "d"));
- inputs2.add(new Tuple2<String, String>("1", "e"));
- inputs2.add(new Tuple2<String, String>("2", "f"));
- inputs2.add(new Tuple2<String, String>("1", "g"));
- inputs2.add(new Tuple2<String, String>("2", "h"));
- inputs2.add(new Tuple2<String, String>("1", "i"));
-
- List<String> expected = new ArrayList<String>();
- expected.add("6");
- expected.add("22");
- expected.add("27");
- expected.add("ace");
- expected.add("bd");
- expected.add("g");
- expected.add("fh");
- expected.add("i");
-
- CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 4L, 3L, keySelector0, keySelector1,
- new MyTimeStamp<Tuple2<String, Integer>>(timestamps1),
- new MyTimeStamp<Tuple2<String, String>>(timestamps2));
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
- assertEquals(expected, result);
- }
-
- @Test
- public void coGroupedWindowReduceTest2() {
-
- List<Long> timestamps1 = Arrays.asList(0L, 0L, 1L, 2L, 2L, 3L, 4L, 4L, 5L, 6L);
- List<Tuple2<String, Integer>> inputs1 = new ArrayList<Tuple2<String, Integer>>();
- inputs1.add(new Tuple2<String, Integer>("a", 1));
- inputs1.add(new Tuple2<String, Integer>("a", 2));
- inputs1.add(new Tuple2<String, Integer>("a", 3));
- inputs1.add(new Tuple2<String, Integer>("a", 4));
- inputs1.add(new Tuple2<String, Integer>("a", 5));
- inputs1.add(new Tuple2<String, Integer>("b", 6));
- inputs1.add(new Tuple2<String, Integer>("a", 7));
- inputs1.add(new Tuple2<String, Integer>("b", 8));
- inputs1.add(new Tuple2<String, Integer>("b", 9));
- inputs1.add(new Tuple2<String, Integer>("b", 10));
-
- List<Long> timestamps2 = Arrays.asList(1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L, 5L);
- List<Tuple2<String, String>> inputs2 = new ArrayList<Tuple2<String, String>>();
- inputs2.add(new Tuple2<String, String>("1", "a"));
- inputs2.add(new Tuple2<String, String>("2", "b"));
- inputs2.add(new Tuple2<String, String>("1", "c"));
- inputs2.add(new Tuple2<String, String>("2", "d"));
- inputs2.add(new Tuple2<String, String>("1", "e"));
- inputs2.add(new Tuple2<String, String>("2", "f"));
- inputs2.add(new Tuple2<String, String>("1", "g"));
- inputs2.add(new Tuple2<String, String>("2", "h"));
- inputs2.add(new Tuple2<String, String>("1", "i"));
-
- List<String> expected = new ArrayList<String>();
- expected.add("15");
- expected.add("6");
- expected.add("16");
- expected.add("23");
- expected.add("7");
- expected.add("27");
- expected.add("ace");
- expected.add("bdf");
- expected.add("egi");
- expected.add("fh");
-
- CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String> invokable = new CoGroupedWindowReduceInvokable<Tuple2<String, Integer>, Tuple2<String, String>, String>(
- new MyCoReduceFunction(), 4L, 3L, 2L, 2L, keySelector0, keySelector1,
- new MyTimeStamp<Tuple2<String, Integer>>(timestamps1),
- new MyTimeStamp<Tuple2<String, String>>(timestamps2));
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs1, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
- assertEquals(expected, result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fcd28fcc/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java
deleted file mode 100644
index 035a021..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowReduceTest.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowReduceInvokable;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoWindowReduceTest {
-
- private static class MyCoReduceFunction implements CoReduceFunction<Integer, String, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce1(Integer value1, Integer value2) {
- return value1 + value2;
- }
-
- @Override
- public String reduce2(String value1, String value2) {
- return value1 + value2;
- }
-
- @Override
- public String map1(Integer value) {
- return value.toString();
- }
-
- @Override
- public String map2(String value) {
- return value;
- }
- }
-
- public static final class MyTimeStamp<T> extends TimestampWrapper<T> {
- private static final long serialVersionUID = 1L;
-
- private Iterator<Long> timestamps;
- private long start;
-
- public MyTimeStamp(List<Long> timestamps) {
- super(null, 0);
- this.timestamps = timestamps.iterator();
- this.start = timestamps.get(0);
- }
-
- @Override
- public long getTimestamp(T value) {
- long ts = timestamps.next();
- return ts;
- }
-
- @Override
- public long getStartTime() {
- return start;
- }
- }
-
- @Test
- public void coWindowReduceTest1() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
-
- List<String> inputs2 = new ArrayList<String>();
- inputs2.add("a");
- inputs2.add("b");
- inputs2.add("c");
- inputs2.add("d");
- inputs2.add("e");
- inputs2.add("f");
- inputs2.add("g");
- inputs2.add("h");
- inputs2.add("i");
-
- List<Long> timestamps1 = Arrays.asList(0L, 2L, 3L, 5L, 7L, 9L, 10L, 11L, 11L, 13L);
- List<Long> timestamps2 = Arrays.asList(0L, 1L, 1L, 2L, 2L, 3L, 3L, 4L, 4L);
-
- CoWindowReduceInvokable<Integer, String, String> invokable = new CoWindowReduceInvokable<Integer, String, String>(
- new MyCoReduceFunction(), 4L, 3L, 4L, 3L, new MyTimeStamp<Integer>(timestamps1),
- new MyTimeStamp<String>(timestamps2));
-
- List<String> expected = new ArrayList<String>();
- expected.add("6");
- expected.add("9");
- expected.add("30");
- expected.add("10");
- expected.add("abcde");
- expected.add("fghi");
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
- assertEquals(expected, result);
-
- }
-
- @Test
- public void coWindowReduceTest2() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
-
- List<String> inputs2 = new ArrayList<String>();
- inputs2.add("a");
- inputs2.add("b");
- inputs2.add("c");
- inputs2.add("d");
- inputs2.add("e");
- inputs2.add("f");
- inputs2.add("g");
- inputs2.add("h");
- inputs2.add("i");
-
- List<Long> timestamps1 = Arrays.asList(0L, 1L, 1L, 1L, 2L, 2L, 3L, 8L, 10L, 11L);
- List<Long> timestamps2 = Arrays.asList(1L, 2L, 4L, 5L, 6L, 9L, 10L, 11L, 13L);
-
- CoWindowReduceInvokable<Integer, String, String> invokable = new CoWindowReduceInvokable<Integer, String, String>(
- new MyCoReduceFunction(), 4L, 3L, 2L, 2L, new MyTimeStamp<Integer>(timestamps1),
- new MyTimeStamp<String>(timestamps2));
-
- List<String> expected = new ArrayList<String>();
- expected.add("28");
- expected.add("18");
- expected.add("8");
- expected.add("27");
- expected.add("ab");
- expected.add("cd");
- expected.add("de");
- expected.add("f");
- expected.add("fgh");
- expected.add("hi");
-
- List<String> result = MockCoContext.createAndExecute(invokable, inputs, inputs2);
-
- Collections.sort(result);
- Collections.sort(expected);
- assertEquals(expected, result);
-
- }
-
-}
[25/36] flink git commit: [streaming] [scala] Restructured streaming
scala project and examples
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
new file mode 100644
index 0000000..0d2a7f5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.Array.canBuildFrom
+import scala.reflect.ClassTag
+
+import org.apache.commons.lang.Validate
+import org.apache.flink.api.common.functions.JoinFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.operators.Keys
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.typeutils.CaseClassSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream}
+import org.apache.flink.streaming.api.datastream.TemporalOperator
+import org.apache.flink.streaming.api.function.co.JoinWindowFunction
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.scala.StreamingConversions._
+import org.apache.flink.streaming.util.keys.KeySelectorUtil
+
+class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
+TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
+
+ override def createNextWindowOperator() = {
+ new StreamJoinOperator.JoinWindow[I1, I2](this)
+ }
+}
+
+object StreamJoinOperator {
+
+ class JoinWindow[I1, I2](private[flink] op: StreamJoinOperator[I1, I2]) {
+
+ private[flink] val type1 = op.input1.getType();
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the fields in the first stream to be used as keys for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where(fields: Int*) = {
+ new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(fields.toArray,type1),type1))
+ }
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the fields in the first stream to be used as keys for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where(firstField: String, otherFields: String*) =
+ new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray,type1),type1))
+
+ /**
+ * Continues a temporal Join transformation by defining
+ * the keyselector function that will be used to extract keys from the first stream
+ * for the join.
+ * The resulting incomplete join can be completed by JoinPredicate.equalTo()
+ * to define the second key.
+ */
+ def where[K: TypeInformation](fun: (I1) => K) = {
+ val keyType = implicitly[TypeInformation[K]]
+ val keyExtractor = new KeySelector[I1, K] {
+ val cleanFun = op.input1.clean(fun)
+ def getKey(in: I1) = cleanFun(in)
+ }
+ new JoinPredicate[I1, I2](op, keyExtractor)
+ }
+
+ }
+
+ class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2],
+ private[flink] val keys1: KeySelector[I1, _]) {
+ private[flink] var keys2: KeySelector[I2, _] = null
+ private[flink] val type2 = op.input2.getType();
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.apply(...)
+ */
+ def equalTo(fields: Int*): JoinedStream[I1, I2] = {
+ finish(KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(fields.toArray,type2),type2))
+ }
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.apply(...)
+ */
+ def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
+ finish(KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray,type2),type2))
+
+ /**
+ * Creates a temporal join transformation by defining the second join key.
+ * The returned transformation wrapes each joined element pair in a tuple2:
+ * (first, second)
+ * To define a custom wrapping, use JoinedStream.apply(...)
+ */
+ def equalTo[K: TypeInformation](fun: (I2) => K): JoinedStream[I1, I2] = {
+ val keyType = implicitly[TypeInformation[K]]
+ val keyExtractor = new KeySelector[I2, K] {
+ val cleanFun = op.input1.clean(fun)
+ def getKey(in: I2) = cleanFun(in)
+ }
+ finish(keyExtractor)
+ }
+
+ private def finish(keys2: KeySelector[I2, _]): JoinedStream[I1, I2] = {
+ this.keys2 = keys2
+ new JoinedStream[I1, I2](this, createJoinOperator())
+ }
+
+ private def createJoinOperator(): JavaStream[(I1, I2)] = {
+
+ val returnType = new CaseClassTypeInfo[(I1, I2)](
+
+ classOf[(I1, I2)], Seq(op.input1.getType, op.input2.getType), Array("_1", "_2")) {
+
+ override def createSerializer: TypeSerializer[(I1, I2)] = {
+ val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
+ for (i <- 0 until getArity) {
+ fieldSerializers(i) = types(i).createSerializer
+ }
+
+ new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
+ override def createInstance(fields: Array[AnyRef]) = {
+ (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
+ }
+ }
+ }
+ }
+
+ return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2))
+ .addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
+ returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+ }
+ }
+
+ class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends
+ DataStream[(I1, I2)](javaStream) {
+
+ private val op = jp.op
+
+ /**
+ * Sets a wrapper for the joined elements. For each joined pair, the result of the
+ * udf call will be emitted.
+ */
+ def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
+
+ val invokable = new CoWindowInvokable[I1, I2, R](
+ clean(getJoinWindowFunction(jp, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
+ op.timeStamp2)
+
+ javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+ invokable)
+
+ javaStream.setType(implicitly[TypeInformation[R]])
+ }
+ }
+
+ private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2],
+ joinFunction: (I1, I2) => R) = {
+ Validate.notNull(joinFunction, "Join function must not be null.")
+
+ val joinFun = new JoinFunction[I1, I2, R] {
+
+ val cleanFun = jp.op.input1.clean(joinFunction)
+
+ override def join(first: I1, second: I2): R = {
+ cleanFun(first, second)
+ }
+ }
+
+ new JoinWindowFunction[I1, I2, R](jp.keys1, jp.keys2, joinFun)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala
new file mode 100644
index 0000000..08ebe03
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamingConversions.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
+import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
+
+object StreamingConversions {
+
+ implicit def javaToScalaStream[R](javaStream: JavaStream[R]): DataStream[R] =
+ new DataStream[R](javaStream)
+
+ implicit def javaToScalaWindowedStream[R](javaWStream: JavaWStream[R]): WindowedDataStream[R] =
+ new WindowedDataStream[R](javaWStream)
+
+ implicit def javaToScalaSplitStream[R](javaStream: SplitJavaStream[R]): SplitDataStream[R] =
+ new SplitDataStream[R](javaStream)
+
+ implicit def javaToScalaConnectedStream[IN1, IN2](javaStream: JavaConStream[IN1, IN2]):
+ ConnectedDataStream[IN1, IN2] = new ConnectedDataStream[IN1, IN2](javaStream)
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
new file mode 100644
index 0000000..9b7edd5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.Array.canBuildFrom
+import scala.collection.JavaConversions.iterableAsScalaIterable
+import scala.reflect.ClassTag
+
+import org.apache.flink.api.common.functions.GroupReduceFunction
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.api.scala._
+import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
+import org.apache.flink.streaming.api.datastream.{WindowedDataStream => JavaWStream}
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
+import org.apache.flink.streaming.api.function.aggregation.SumFunction
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.scala.StreamingConversions._
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
+import org.apache.flink.streaming.api.windowing.helper._
+import org.apache.flink.util.Collector
+
+class WindowedDataStream[T](javaStream: JavaWStream[T]) {
+
+ /**
+ * Defines the slide size (trigger frequency) for the windowed data stream.
+ * This controls how often the user defined function will be triggered on
+ * the window.
+ */
+ def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
+ javaStream.every(windowingHelper: _*)
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * field positions. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy(fields: Int*): WindowedDataStream[T] = javaStream.groupBy(fields: _*)
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * field expressions. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy(firstField: String, otherFields: String*): WindowedDataStream[T] =
+ javaStream.groupBy(firstField +: otherFields.toArray: _*)
+
+ /**
+ * Groups the elements of the WindowedDataStream using the given
+ * KeySelector function. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * DataStream.window(...) operator on an already grouped data stream.
+ *
+ */
+ def groupBy[K: TypeInformation](fun: T => K): WindowedDataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ javaStream.groupBy(keyExtractor)
+ }
+
+ /**
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.
+ *
+ */
+ def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
+ if (reducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ javaStream.reduce(reducer)
+ }
+
+ /**
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.
+ *
+ */
+ def reduce(fun: (T, T) => T): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ val reducer = new ReduceFunction[T] {
+ val cleanFun = clean(fun)
+ def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by reducing
+ * the current window at every trigger. In contrast with the simple binary reduce operator,
+ * groupReduce exposes the whole window through the Iterable interface.
+ * </br>
+ * </br>
+ * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+ */
+ def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
+ DataStream[R] = {
+ if (reducer == null) {
+ throw new NullPointerException("GroupReduce function must not be null.")
+ }
+ javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]])
+ }
+
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by reducing
+ * the current window at every trigger. In contrast with the simple binary reduce operator,
+ * groupReduce exposes the whole window through the Iterable interface.
+ * </br>
+ * </br>
+ * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+ */
+ def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
+ DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("GroupReduce function must not be null.")
+ }
+ val reducer = new GroupReduceFunction[T, R] {
+ val cleanFun = clean(fun)
+ def reduce(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
+ }
+ reduceGroup(reducer)
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum of the elements in the window at
+ * the given position.
+ *
+ */
+ def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
+
+ /**
+ * Applies an aggregation that that gives the minimum of the elements in the window at
+ * the given position.
+ *
+ */
+ def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
+
+ /**
+ * Applies an aggregation that sums the elements in the window at the given position.
+ *
+ */
+ def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
+
+ /**
+ * Applies an aggregation that that gives the maximum element of the window by
+ * the given position. When equality, returns the first.
+ *
+ */
+ def maxBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MAXBY,
+ position, first)
+
+ /**
+ * Applies an aggregation that that gives the minimum element of the window by
+ * the given position. When equality, returns the first.
+ *
+ */
+ def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType.MINBY,
+ position, first)
+
+ def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true):
+ DataStream[T] = {
+
+ val jStream = javaStream.asInstanceOf[JavaWStream[Product]]
+ val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
+
+ val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
+
+ val reducer = aggregationType match {
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(
+ outType.getTypeAt(position).getTypeClass()));
+ case _ => new agg.ProductComparableAggregator(aggregationType, first)
+ }
+
+ new DataStream[Product](jStream.reduce(reducer)).asInstanceOf[DataStream[T]]
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
new file mode 100644
index 0000000..c5a3e07
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Delta.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala.windowing
+
+import org.apache.flink.streaming.api.windowing.helper.{ Delta => JavaDelta }
+import org.apache.commons.lang.Validate
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean;
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction
+
+object Delta {
+
+ /**
+ * Creates a delta helper representing a delta trigger or eviction policy.
+ * </br></br> This policy calculates a delta between the data point which
+ * triggered last and the currently arrived data point. It triggers if the
+ * delta is higher than a specified threshold. </br></br> In case it gets
+ * used for eviction, this policy starts from the first element of the
+ * buffer and removes all elements from the buffer which have a higher delta
+ * then the threshold. As soon as there is an element with a lower delta,
+ * the eviction stops.
+ */
+ def of[T](threshold: Double, deltaFunction: (T, T) => Double, initVal: T): JavaDelta[T] = {
+ Validate.notNull(deltaFunction, "Delta function must not be null")
+ val df = new DeltaFunction[T] {
+ val cleanFun = clean(deltaFunction)
+ override def getDelta(first: T, second: T) = cleanFun(first, second)
+ }
+ JavaDelta.of(threshold, df, initVal)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
new file mode 100644
index 0000000..457d10d
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/windowing/Time.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala.windowing
+
+import java.util.concurrent.TimeUnit
+import org.apache.flink.streaming.api.windowing.helper.{ Time => JavaTime }
+
+import org.apache.flink.api.scala.ClosureCleaner
+import org.apache.commons.net.ntp.TimeStamp
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp
+import org.apache.commons.lang.Validate
+
+object Time {
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using System time.
+ *
+ */
+ def of(windowSize: Long, timeUnit: TimeUnit): JavaTime[_] =
+ JavaTime.of(windowSize, timeUnit)
+
+ /**
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using a user defined timestamp extractor.
+ *
+ */
+ def of[R](windowSize: Long, timestamp: R => Long, startTime: Long = 0): JavaTime[R] = {
+ Validate.notNull(timestamp, "Timestamp must not be null.")
+ val ts = new Timestamp[R] {
+ val fun = clean(timestamp, true)
+ override def getTimestamp(in: R) = fun(in)
+ }
+ JavaTime.of(windowSize, ts, startTime)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/pom.xml b/flink-addons/flink-streaming/pom.xml
index 142bd95..5d25e53 100644
--- a/flink-addons/flink-streaming/pom.xml
+++ b/flink-addons/flink-streaming/pom.xml
@@ -35,6 +35,7 @@ under the License.
<modules>
<module>flink-streaming-core</module>
+ <module>flink-streaming-scala</module>
<module>flink-streaming-examples</module>
<module>flink-streaming-connectors</module>
</modules>
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
deleted file mode 100644
index e39fb11..0000000
--- a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/TopSpeedWindowing.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.scala.streaming.windowing
-
-
-import java.util.concurrent.TimeUnit._
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment
-import org.apache.flink.api.scala.streaming.windowing.Delta
-import org.apache.flink.streaming.api.windowing.helper.Time
-import org.apache.flink.util.Collector
-
-import scala.math.{max, min}
-import scala.util.Random
-
-/**
- * An example of grouped stream windowing where different eviction and
- * trigger policies can be used.A source fetches events from cars
- * every 1 sec containing their id, their current speed (kmh),
- * overall elapsed distance (m) and a timestamp. The streaming
- * example triggers the top speed of each car every x meters elapsed
- * for the last y seconds.
- */
-object TopSpeedWindowing {
-
- case class CarSpeed(carId: Int, speed: Int, distance: Double, time: Long)
-
- def main(args: Array[String]) {
- if (!parseParameters(args)) {
- return
- }
-
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- val cars = env.addSource(carSource _).groupBy("carId")
- .window(Time.of(evictionSec, SECONDS))
- .every(Delta.of[CarSpeed](triggerMeters,
- (oldSp,newSp) => newSp.distance-oldSp.distance, CarSpeed(0,0,0,0)))
- .reduce((x, y) => if (x.speed > y.speed) x else y)
-
- cars print
-
- env.execute("TopSpeedWindowing")
-
- }
-
- def carSource(out: Collector[CarSpeed]) = {
-
- val speeds = new Array[Int](numOfCars)
- val distances = new Array[Double](numOfCars)
-
- while (true) {
- Thread sleep 1000
- for (i <- 0 until speeds.length) {
- speeds(i) = if (Random.nextBoolean) min(100, speeds(i) + 5) else max(0, speeds(i) - 5)
- distances(i) += speeds(i) / 3.6d
- out.collect(new CarSpeed(i, speeds(i), distances(i), System.currentTimeMillis))
- }
- }
- }
-
- def parseParameters(args: Array[String]): Boolean = {
- if (args.length > 0) {
- if (args.length == 3) {
- numOfCars = args(0).toInt
- evictionSec = args(1).toInt
- triggerMeters = args(2).toDouble
- }
- else {
- System.err.println("Usage: TopSpeedWindowing <numCars> <evictSec> <triggerMeters>")
- false
- }
- }
- true
- }
-
- var numOfCars = 2
- var evictionSec = 10
- var triggerMeters = 50d
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala b/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
deleted file mode 100644
index caf5eb9..0000000
--- a/flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/streaming/windowing/WindowJoin.scala
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.scala.streaming.windowing
-
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment
-import org.apache.flink.util.Collector
-import scala.util.Random
-
-object WindowJoin {
-
- case class Name(id: Long, name: String)
- case class Age(id: Long, age: Int)
- case class Person(name: String, age: Long)
-
- def main(args: Array[String]) {
-
- val env = StreamExecutionEnvironment.getExecutionEnvironment
-
- //Create streams for names and ages by mapping the inputs to the corresponding objects
- val names = env.addSource(nameStream _).map(x => Name(x._1, x._2))
- val ages = env.addSource(ageStream _).map(x => Age(x._1, x._2))
-
- //Join the two input streams by id on the last second and create new Person objects
- //containing both name and age
- val joined =
- names.join(ages).onWindow(1000)
- .where("id").equalTo("id") { (n, a) => Person(n.name, a.age) }
-
- joined print
-
- env.execute("WindowJoin")
- }
-
- //Stream source for generating (id, name) pairs
- def nameStream(out: Collector[(Long, String)]) = {
- val names = Array("tom", "jerry", "alice", "bob", "john", "grace")
-
- for (i <- 1 to 10000) {
- if (i % 100 == 0) Thread.sleep(1000) else {
- out.collect((i, names(Random.nextInt(names.length))))
- }
- }
- }
-
- //Stream source for generating (id, age) pairs
- def ageStream(out: Collector[(Long, Int)]) = {
- for (i <- 1 to 10000) {
- if (i % 100 == 0) Thread.sleep(1000) else {
- out.collect((i, Random.nextInt(90)))
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-scala/pom.xml b/flink-scala/pom.xml
index f55fc71..e2d71e1 100644
--- a/flink-scala/pom.xml
+++ b/flink-scala/pom.xml
@@ -50,12 +50,6 @@ under the License.
<artifactId>flink-compiler</artifactId>
<version>${project.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-core</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.scala-lang</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java b/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java
deleted file mode 100644
index 2f587d7..0000000
--- a/flink-scala/src/main/java/org/apache/flink/api/scala/streaming/ScalaStreamingAggregator.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
-import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
-import org.apache.flink.streaming.api.function.aggregation.SumFunction;
-
-import scala.Product;
-
-public class ScalaStreamingAggregator<IN extends Product> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- TupleSerializerBase<IN> serializer;
- Object[] fields;
- int length;
- int position;
-
- public ScalaStreamingAggregator(TypeSerializer<IN> serializer, int pos) {
- this.serializer = (TupleSerializerBase<IN>) serializer;
- this.length = this.serializer.getArity();
- this.fields = new Object[this.length];
- this.position = pos;
- }
-
- public class Sum extends AggregationFunction<IN> {
- private static final long serialVersionUID = 1L;
- SumFunction sumFunction;
-
- public Sum(SumFunction func) {
- super(ScalaStreamingAggregator.this.position);
- this.sumFunction = func;
- }
-
- @Override
- public IN reduce(IN value1, IN value2) throws Exception {
- for (int i = 0; i < length; i++) {
- fields[i] = value2.productElement(i);
- }
-
- fields[position] = sumFunction.add(fields[position], value1.productElement(position));
-
- return serializer.createInstance(fields);
- }
- }
-
- public class ProductComparableAggregator extends ComparableAggregator<IN> {
-
- private static final long serialVersionUID = 1L;
-
- public ProductComparableAggregator(AggregationFunction.AggregationType aggregationType,
- boolean first) {
- super(ScalaStreamingAggregator.this.position, aggregationType, first);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public IN reduce(IN value1, IN value2) throws Exception {
- Object v1 = value1.productElement(position);
- Object v2 = value2.productElement(position);
-
- int c = comparator.isExtremal((Comparable<Object>) v1, v2);
-
- if (byAggregate) {
- if (c == 1) {
- return value1;
- }
- if (first) {
- if (c == 0) {
- return value1;
- }
- }
-
- return value2;
- } else {
- for (int i = 0; i < length; i++) {
- fields[i] = value2.productElement(i);
- }
-
- if (c == 1) {
- fields[position] = v1;
- }
-
- return serializer.createInstance(fields);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
deleted file mode 100644
index 985e512..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/ConnectedDataStream.scala
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-
-import java.util
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment._
-import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaCStream }
-import org.apache.flink.streaming.api.function.co.{ CoFlatMapFunction, CoMapFunction, CoReduceFunction, CoWindowFunction }
-import org.apache.flink.streaming.api.invokable.operator.co.{ CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable }
-import org.apache.flink.util.Collector
-import org.apache.flink.api.scala.streaming.StreamingConversions._
-
-import scala.collection.JavaConversions._
-import scala.reflect.ClassTag
-
-class ConnectedDataStream[IN1, IN2](javaStream: JavaCStream[IN1, IN2]) {
-
- /**
- * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
- * the output to a common type. The transformation calls a
- * @param fun1 for each element of the first input and
- * @param fun2 for each element of the second input. Each
- * CoMapFunction call returns exactly one element.
- *
- * The CoMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed { @link DataStream}
- */
- def map[R: TypeInformation: ClassTag](fun1: IN1 => R, fun2: IN2 => R):
- DataStream[R] = {
- if (fun1 == null || fun2 == null) {
- throw new NullPointerException("Map function must not be null.")
- }
- val comapper = new CoMapFunction[IN1, IN2, R] {
- def map1(in1: IN1): R = clean(fun1)(in1)
- def map2(in2: IN2): R = clean(fun2)(in2)
- }
-
- new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
- new CoMapInvokable[IN1, IN2, R](comapper)))
- }
-
- /**
- * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
- * the output to a common type. The transformation calls a
- * {@link CoMapFunction#map1} for each element of the first input and
- * {@link CoMapFunction#map2} for each element of the second input. Each
- * CoMapFunction call returns exactly one element. The user can also extend
- * {@link RichCoMapFunction} to gain access to other features provided by
- * the {@link RichFuntion} interface.
- *
- * @param coMapper
- * The CoMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed { @link DataStream}
- */
- def map[R: TypeInformation: ClassTag](coMapper: CoMapFunction[IN1, IN2, R]):
- DataStream[R] = {
- if (coMapper == null) {
- throw new NullPointerException("Map function must not be null.")
- }
-
- new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
- new CoMapInvokable[IN1, IN2, R](coMapper)))
- }
-
- /**
- * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
- * maps the output to a common type. The transformation calls a
- * {@link CoFlatMapFunction#flatMap1} for each element of the first input
- * and {@link CoFlatMapFunction#flatMap2} for each element of the second
- * input. Each CoFlatMapFunction call returns any number of elements
- * including none. The user can also extend {@link RichFlatMapFunction} to
- * gain access to other features provided by the {@link RichFuntion}
- * interface.
- *
- * @param coFlatMapper
- * The CoFlatMapFunction used to jointly transform the two input
- * DataStreams
- * @return The transformed { @link DataStream}
- */
- def flatMap[R: TypeInformation: ClassTag](coFlatMapper: CoFlatMapFunction[IN1, IN2, R]):
- DataStream[R] = {
- if (coFlatMapper == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- new DataStream[R](javaStream.addCoFunction("flatMap", implicitly[TypeInformation[R]],
- new CoFlatMapInvokable[IN1, IN2, R](coFlatMapper)))
- }
-
- /**
- * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
- * maps the output to a common type. The transformation calls a
- * @param fun1 for each element of the first input
- * and @param fun2 for each element of the second
- * input. Each CoFlatMapFunction call returns any number of elements
- * including none.
- *
- * @return The transformed { @link DataStream}
- */
- def flatMap[R: TypeInformation: ClassTag](fun1: (IN1, Collector[R]) => Unit,
- fun2: (IN2, Collector[R]) => Unit): DataStream[R] = {
- if (fun1 == null || fun2 == null) {
- throw new NullPointerException("FlatMap functions must not be null.")
- }
- val flatMapper = new CoFlatMapFunction[IN1, IN2, R] {
- def flatMap1(value: IN1, out: Collector[R]): Unit = clean(fun1)(value, out)
- def flatMap2(value: IN2, out: Collector[R]): Unit = clean(fun2)(value, out)
- }
- flatMap(flatMapper)
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 according to keyPosition1 and keyPosition2. Used for
- * applying function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param keyPosition1
- * The field used to compute the hashcode of the elements in the
- * first input stream.
- * @param keyPosition2
- * The field used to compute the hashcode of the elements in the
- * second input stream.
- * @return @return The transformed { @link ConnectedDataStream}
- */
- def groupBy(keyPosition1: Int, keyPosition2: Int): ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(keyPosition1, keyPosition2)
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 according to keyPositions1 and keyPositions2. Used for
- * applying function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param keyPositions1
- * The fields used to group the first input stream.
- * @param keyPositions2
- * The fields used to group the second input stream.
- * @return @return The transformed { @link ConnectedDataStream}
- */
- def groupBy(keyPositions1: Array[Int], keyPositions2: Array[Int]):
- ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(keyPositions1, keyPositions2)
- }
-
- /**
- * GroupBy operation for connected data stream using key expressions. Groups
- * the elements of input1 and input2 according to field1 and field2. A field
- * expression is either the name of a public field or a getter method with
- * parentheses of the {@link DataStream}S underlying type. A dot can be used
- * to drill down into objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field1
- * The grouping expression for the first input
- * @param field2
- * The grouping expression for the second input
- * @return The grouped { @link ConnectedDataStream}
- */
- def groupBy(field1: String, field2: String): ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(field1, field2)
- }
-
- /**
- * GroupBy operation for connected data stream using key expressions. Groups
- * the elements of input1 and input2 according to fields1 and fields2. A
- * field expression is either the name of a public field or a getter method
- * with parentheses of the {@link DataStream}S underlying type. A dot can be
- * used to drill down into objects, as in {@code "field1.getInnerField2()" }
- * .
- *
- * @param fields1
- * The grouping expressions for the first input
- * @param fields2
- * The grouping expressions for the second input
- * @return The grouped { @link ConnectedDataStream}
- */
- def groupBy(fields1: Array[String], fields2: Array[String]):
- ConnectedDataStream[IN1, IN2] = {
- javaStream.groupBy(fields1, fields2)
- }
-
- /**
- * GroupBy operation for connected data stream. Groups the elements of
- * input1 and input2 using fun1 and fun2. Used for applying
- * function on grouped data streams for example
- * {@link ConnectedDataStream#reduce}
- *
- * @param fun1
- * The function used for grouping the first input
- * @param fun2
- * The function used for grouping the second input
- * @return @return The transformed { @link ConnectedDataStream}
- */
- def groupBy[K: TypeInformation](fun1: IN1 => _, fun2: IN2 => _):
- ConnectedDataStream[IN1, IN2] = {
-
- val keyExtractor1 = new KeySelector[IN1, Any] {
- def getKey(in: IN1) = clean(fun1)(in)
- }
- val keyExtractor2 = new KeySelector[IN2, Any] {
- def getKey(in: IN2) = clean(fun2)(in)
- }
-
- javaStream.groupBy(keyExtractor1, keyExtractor2)
- }
-
- /**
- * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
- * the outputs to a common type. If the {@link ConnectedDataStream} is
- * batched or windowed then the reduce transformation is applied on every
- * sliding batch/window of the data stream. If the connected data stream is
- * grouped then the reducer is applied on every group of elements sharing
- * the same key. This type of reduce is much faster than reduceGroup since
- * the reduce function can be applied incrementally.
- *
- * @param coReducer
- * The { @link CoReduceFunction} that will be called for every
- * element of the inputs.
- * @return The transformed { @link DataStream}.
- */
- def reduce[R: TypeInformation: ClassTag](coReducer: CoReduceFunction[IN1, IN2, R]):
- DataStream[R] = {
- if (coReducer == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
-
- new DataStream[R](javaStream.addCoFunction("coReduce", implicitly[TypeInformation[R]],
- new CoReduceInvokable[IN1, IN2, R](coReducer)))
- }
-
- /**
- * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
- * the outputs to a common type. If the {@link ConnectedDataStream} is
- * batched or windowed then the reduce transformation is applied on every
- * sliding batch/window of the data stream. If the connected data stream is
- * grouped then the reducer is applied on every group of elements sharing
- * the same key. This type of reduce is much faster than reduceGroup since
- * the reduce function can be applied incrementally.
- *
- * @param reducer1
- * @param reducer2
- * @param mapper1
- * @param mapper2
- *
- * @return The transformed { @link DataStream}.
- */
- def reduce[R: TypeInformation: ClassTag](reducer1: (IN1, IN1) => IN1,
- reducer2: (IN2, IN2) => IN2,mapper1: IN1 => R, mapper2: IN2 => R): DataStream[R] = {
- if (mapper1 == null || mapper2 == null) {
- throw new NullPointerException("Map functions must not be null.")
- }
- if (reducer1 == null || reducer2 == null) {
- throw new NullPointerException("Reduce functions must not be null.")
- }
-
- val reducer = new CoReduceFunction[IN1, IN2, R] {
- def reduce1(value1: IN1, value2: IN1): IN1 = clean(reducer1)(value1, value2)
- def map2(value: IN2): R = clean(mapper2)(value)
- def reduce2(value1: IN2, value2: IN2): IN2 = clean(reducer2)(value1, value2)
- def map1(value: IN1): R = clean(mapper1)(value)
- }
- reduce(reducer)
- }
-
- /**
- * Applies a CoWindow transformation on the connected DataStreams. The
- * transformation calls the {@link CoWindowFunction#coWindow} method for for
- * time aligned windows of the two data streams. System time is used as
- * default to compute windows.
- *
- * @param coWindowFunction
- * The { @link CoWindowFunction} that will be applied for the time
- * windows.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- *
- * @return The transformed { @link DataStream}.
- */
- def windowReduce[R: TypeInformation: ClassTag](coWindowFunction:
- CoWindowFunction[IN1, IN2, R], windowSize: Long, slideInterval: Long) = {
- if (coWindowFunction == null) {
- throw new NullPointerException("CoWindow function must no be null")
- }
-
- javaStream.windowReduce(coWindowFunction, windowSize, slideInterval)
- }
-
- /**
- * Applies a CoWindow transformation on the connected DataStreams. The
- * transformation calls the {@link CoWindowFunction#coWindow} method for for
- * time aligned windows of the two data streams. System time is used as
- * default to compute windows.
- *
- * @param coWindower
- * The coWindowing function to be applied for the time windows.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- *
- * @return The transformed { @link DataStream}.
- */
- def windowReduce[R: TypeInformation: ClassTag](coWindower: (Seq[IN1], Seq[IN2],
- Collector[R]) => Unit, windowSize: Long, slideInterval: Long) = {
- if (coWindower == null) {
- throw new NullPointerException("CoWindow function must no be null")
- }
-
- val coWindowFun = new CoWindowFunction[IN1, IN2, R] {
- def coWindow(first: util.List[IN1], second: util.List[IN2],
- out: Collector[R]): Unit = clean(coWindower)(first, second, out)
- }
-
- javaStream.windowReduce(coWindowFun, windowSize, slideInterval)
- }
-
- /**
- * Returns the first {@link DataStream}.
- *
- * @return The first DataStream.
- */
- def getFirst(): DataStream[IN1] = {
- javaStream.getFirst
- }
-
- /**
- * Returns the second {@link DataStream}.
- *
- * @return The second DataStream.
- */
- def getSecond(): DataStream[IN2] = {
- javaStream.getSecond
- }
-
- /**
- * Gets the type of the first input
- *
- * @return The type of the first input
- */
- def getInputType1(): TypeInformation[IN1] = {
- javaStream.getInputType1
- }
-
- /**
- * Gets the type of the second input
- *
- * @return The type of the second input
- */
- def getInputType2(): TypeInformation[IN2] = {
- javaStream.getInputType2
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
deleted file mode 100644
index ccfd176..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ /dev/null
@@ -1,558 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream,
- SingleOutputStreamOperator, GroupedDataStream}
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import scala.reflect.ClassTag
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.functions.MapFunction
-import org.apache.flink.streaming.api.invokable.operator.MapInvokable
-import org.apache.flink.util.Collector
-import org.apache.flink.api.common.functions.FlatMapFunction
-import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
-import org.apache.flink.api.common.functions.ReduceFunction
-import org.apache.flink.streaming.api.invokable.StreamInvokable
-import org.apache.flink.streaming.api.invokable.operator.GroupedReduceInvokable
-import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable
-import org.apache.flink.api.common.functions.ReduceFunction
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.common.functions.FilterFunction
-import org.apache.flink.streaming.api.function.sink.SinkFunction
-import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy
-import org.apache.flink.streaming.api.collector.OutputSelector
-import scala.collection.JavaConversions._
-import java.util.HashMap
-import org.apache.flink.streaming.api.function.aggregation.SumFunction
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.api.scala.streaming.StreamingConversions._
-
-class DataStream[T](javaStream: JavaStream[T]) {
-
- /**
- * Gets the underlying java DataStream object.
- */
- def getJavaStream: JavaStream[T] = javaStream
-
- /**
- * Sets the degree of parallelism of this operation. This must be greater than 1.
- */
- def setParallelism(dop: Int): DataStream[T] = {
- javaStream match {
- case ds: SingleOutputStreamOperator[_, _] => ds.setParallelism(dop)
- case _ =>
- throw new UnsupportedOperationException("Operator " + javaStream.toString + " cannot " +
- "have " +
- "parallelism.")
- }
- this
- }
-
- /**
- * Returns the degree of parallelism of this operation.
- */
- def getParallelism: Int = javaStream match {
- case op: SingleOutputStreamOperator[_, _] => op.getParallelism
- case _ =>
- throw new UnsupportedOperationException("Operator " + javaStream.toString + " does not have" +
- " " +
- "parallelism.")
- }
-
- /**
- * Creates a new DataStream by merging DataStream outputs of
- * the same type with each other. The DataStreams merged using this operator
- * will be transformed simultaneously.
- *
- */
- def merge(dataStreams: DataStream[T]*): DataStream[T] =
- javaStream.merge(dataStreams.map(_.getJavaStream): _*)
-
- /**
- * Creates a new ConnectedDataStream by connecting
- * DataStream outputs of different type with each other. The
- * DataStreams connected using this operators can be used with CoFunctions.
- *
- */
- def connect[T2](dataStream: DataStream[T2]): ConnectedDataStream[T, T2] =
- javaStream.connect(dataStream.getJavaStream)
-
- /**
- * Groups the elements of a DataStream by the given key positions (for tuple/array types) to
- * be used with grouped operators like grouped reduce or grouped aggregations
- *
- */
- def groupBy(fields: Int*): DataStream[T] = javaStream.groupBy(fields: _*)
-
- /**
- * Groups the elements of a DataStream by the given field expressions to
- * be used with grouped operators like grouped reduce or grouped aggregations
- *
- */
- def groupBy(firstField: String, otherFields: String*): DataStream[T] =
- javaStream.groupBy(firstField +: otherFields.toArray: _*)
-
- /**
- * Groups the elements of a DataStream by the given K key to
- * be used with grouped operators like grouped reduce or grouped aggregations
- *
- */
- def groupBy[K: TypeInformation](fun: T => K): DataStream[T] = {
-
- val keyExtractor = new KeySelector[T, K] {
- val cleanFun = clean(fun)
- def getKey(in: T) = cleanFun(in)
- }
- javaStream.groupBy(keyExtractor)
- }
-
- /**
- * Sets the partitioning of the DataStream so that the output is
- * partitioned by the selected fields. This setting only effects the how the outputs will be
- * distributed between the parallel instances of the next processing operator.
- *
- */
- def partitionBy(fields: Int*): DataStream[T] =
- javaStream.partitionBy(fields: _*)
-
- /**
- * Sets the partitioning of the DataStream so that the output is
- * partitioned by the selected fields. This setting only effects the how the outputs will be
- * distributed between the parallel instances of the next processing operator.
- *
- */
- def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
- javaStream.partitionBy(firstField +: otherFields.toArray: _*)
-
- /**
- * Sets the partitioning of the DataStream so that the output is
- * partitioned by the given Key. This setting only effects the how the outputs will be
- * distributed between the parallel instances of the next processing operator.
- *
- */
- def partitionBy[K: TypeInformation](fun: T => K): DataStream[T] = {
-
- val keyExtractor = new KeySelector[T, K] {
- val cleanFun = clean(fun)
- def getKey(in: T) = cleanFun(in)
- }
- javaStream.partitionBy(keyExtractor)
- }
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are broadcasted to every parallel instance of the next component. This
- * setting only effects the how the outputs will be distributed between the
- * parallel instances of the next processing operator.
- *
- */
- def broadcast: DataStream[T] = javaStream.broadcast()
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are shuffled to the next component. This setting only effects the how the
- * outputs will be distributed between the parallel instances of the next
- * processing operator.
- *
- */
- def shuffle: DataStream[T] = javaStream.shuffle()
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are forwarded to the local subtask of the next component (whenever
- * possible). This is the default partitioner setting. This setting only
- * effects the how the outputs will be distributed between the parallel
- * instances of the next processing operator.
- *
- */
- def forward: DataStream[T] = javaStream.forward()
-
- /**
- * Sets the partitioning of the DataStream so that the output tuples
- * are distributed evenly to the next component.This setting only effects
- * the how the outputs will be distributed between the parallel instances of
- * the next processing operator.
- *
- */
- def distribute: DataStream[T] = javaStream.distribute()
-
- /**
- * Initiates an iterative part of the program that creates a loop by feeding
- * back data streams. To create a streaming iteration the user needs to define
- * a transformation that creates two DataStreams.The first one one is the output
- * that will be fed back to the start of the iteration and the second is the output
- * stream of the iterative part.
- * <p>
- * stepfunction: initialStream => (feedback, output)
- * <p>
- * A common pattern is to use output splitting to create feedback and output DataStream.
- * Please refer to the .split(...) method of the DataStream
- * <p>
- * By default a DataStream with iteration will never terminate, but the user
- * can use the maxWaitTime parameter to set a max waiting time for the iteration head.
- * If no data received in the set time the stream terminates.
- *
- *
- */
- def iterate(stepFunction: DataStream[T] => (DataStream[T], DataStream[T]), maxWaitTimeMillis:
- Long = 0): DataStream[T] = {
- val iterativeStream = javaStream.iterate(maxWaitTimeMillis)
-
- val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
- iterativeStream.closeWith(feedback.getJavaStream)
- output
- }
-
- /**
- * Applies an aggregation that that gives the current maximum of the data stream at
- * the given position.
- *
- */
- def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
-
- /**
- * Applies an aggregation that that gives the current minimum of the data stream at
- * the given position.
- *
- */
- def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
-
- /**
- * Applies an aggregation that sums the data stream at the given position.
- *
- */
- def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
-
- /**
- * Applies an aggregation that that gives the current minimum element of the data stream by
- * the given position. When equality, the user can set to get the first or last element with
- * the minimal value.
- *
- */
- def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType
- .MINBY, position, first)
-
- /**
- * Applies an aggregation that that gives the current maximum element of the data stream by
- * the given position. When equality, the user can set to get the first or last element with
- * the maximal value.
- *
- */
- def maxBy(position: Int, first: Boolean = true): DataStream[T] =
- aggregate(AggregationType.MAXBY, position, first)
-
- private def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true):
- DataStream[T] = {
-
- val jStream = javaStream.asInstanceOf[JavaStream[Product]]
- val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
-
- val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
-
- val reducer = aggregationType match {
- case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).
- getTypeClass()));
- case _ => new agg.ProductComparableAggregator(aggregationType, first)
- }
-
- val invokable = jStream match {
- case groupedStream: GroupedDataStream[_] => new GroupedReduceInvokable(reducer,
- groupedStream.getKeySelector())
- case _ => new StreamReduceInvokable(reducer)
- }
- new DataStream[Product](jStream.transform("aggregation", jStream.getType(),
- invokable)).asInstanceOf[DataStream[T]]
- }
-
- /**
- * Creates a new DataStream containing the current number (count) of
- * received records.
- *
- */
- def count: DataStream[Long] = new DataStream[java.lang.Long](
- javaStream.count()).asInstanceOf[DataStream[Long]]
-
- /**
- * Creates a new DataStream by applying the given function to every element of this DataStream.
- */
- def map[R: TypeInformation: ClassTag](fun: T => R): DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("Map function must not be null.")
- }
- val mapper = new MapFunction[T, R] {
- val cleanFun = clean(fun)
- def map(in: T): R = cleanFun(in)
- }
-
- javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element of this DataStream.
- */
- def map[R: TypeInformation: ClassTag](mapper: MapFunction[T, R]): DataStream[R] = {
- if (mapper == null) {
- throw new NullPointerException("Map function must not be null.")
- }
-
- javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element and flattening
- * the results.
- */
- def flatMap[R: TypeInformation: ClassTag](flatMapper: FlatMapFunction[T, R]): DataStream[R] = {
- if (flatMapper == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- javaStream.transform("flatMap", implicitly[TypeInformation[R]],
- new FlatMapInvokable[T, R](flatMapper))
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element and flattening
- * the results.
- */
- def flatMap[R: TypeInformation: ClassTag](fun: (T, Collector[R]) => Unit): DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- val flatMapper = new FlatMapFunction[T, R] {
- val cleanFun = clean(fun)
- def flatMap(in: T, out: Collector[R]) { cleanFun(in, out) }
- }
- flatMap(flatMapper)
- }
-
- /**
- * Creates a new DataStream by applying the given function to every element and flattening
- * the results.
- */
- def flatMap[R: TypeInformation: ClassTag](fun: T => TraversableOnce[R]): DataStream[R] = {
- if (fun == null) {
- throw new NullPointerException("FlatMap function must not be null.")
- }
- val flatMapper = new FlatMapFunction[T, R] {
- val cleanFun = clean(fun)
- def flatMap(in: T, out: Collector[R]) { cleanFun(in) foreach out.collect }
- }
- flatMap(flatMapper)
- }
-
- /**
- * Creates a new [[DataStream]] by reducing the elements of this DataStream
- * using an associative reduce function.
- */
- def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
- if (reducer == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- javaStream match {
- case ds: GroupedDataStream[_] => javaStream.transform("reduce",
- javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector()))
- case _ => javaStream.transform("reduce", javaStream.getType(),
- new StreamReduceInvokable[T](reducer))
- }
- }
-
- /**
- * Creates a new [[DataStream]] by reducing the elements of this DataStream
- * using an associative reduce function.
- */
- def reduce(fun: (T, T) => T): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Reduce function must not be null.")
- }
- val reducer = new ReduceFunction[T] {
- val cleanFun = clean(fun)
- def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
- }
- reduce(reducer)
- }
-
- /**
- * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
- */
- def filter(filter: FilterFunction[T]): DataStream[T] = {
- if (filter == null) {
- throw new NullPointerException("Filter function must not be null.")
- }
- javaStream.filter(filter)
- }
-
- /**
- * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
- */
- def filter(fun: T => Boolean): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Filter function must not be null.")
- }
- val filter = new FilterFunction[T] {
- val cleanFun = clean(fun)
- def filter(in: T) = cleanFun(in)
- }
- this.filter(filter)
- }
-
- /**
- * Create a WindowedDataStream that can be used to apply
- * transformation like .reduce(...) or aggregations on
- * preset chunks(windows) of the data stream. To define the windows one or
- * more WindowingHelper-s such as Time, Count and
- * Delta can be used.</br></br> When applied to a grouped data
- * stream, the windows (evictions) and slide sizes (triggers) will be
- * computed on a per group basis. </br></br> For more advanced control over
- * the trigger and eviction policies please use to
- * window(List(triggers), List(evicters))
- */
- def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
- javaStream.window(windowingHelper: _*)
-
- /**
- * Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
- * Windowing can be used to apply transformation like .reduce(...) or aggregations on
- * preset chunks(windows) of the data stream.</br></br>For most common
- * use-cases please refer to window(WindowingHelper[_]*)
- *
- */
- def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]):
- WindowedDataStream[T] = javaStream.window(triggers, evicters)
-
- /**
- *
- * Operator used for directing tuples to specific named outputs using an
- * OutputSelector. Calling this method on an operator creates a new
- * SplitDataStream.
- */
- def split(selector: OutputSelector[T]): SplitDataStream[T] = javaStream match {
- case op: SingleOutputStreamOperator[_, _] => op.split(selector)
- case _ =>
- throw new UnsupportedOperationException("Operator " + javaStream.toString + " can not be " +
- "split.")
- }
-
- /**
- * Creates a new SplitDataStream that contains only the elements satisfying the
- * given output selector predicate.
- */
- def split(fun: T => String): SplitDataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("OutputSelector must not be null.")
- }
- val selector = new OutputSelector[T] {
- val cleanFun = clean(fun)
- def select(in: T): java.lang.Iterable[String] = {
- List(cleanFun(in))
- }
- }
- split(selector)
- }
-
- /**
- * Initiates a temporal Join transformation that joins the elements of two
- * data streams on key equality over a specified time window.
- *
- * This method returns a StreamJoinOperator on which the
- * .onWindow(..) should be called to define the
- * window, and then the .where(..) and .equalTo(..) methods can be used to defin
- * the join keys.</p> The user can also use the apply method of the returned JoinedStream
- * to use custom join function.
- *
- */
- def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] =
- new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
-
- /**
- * Initiates a temporal cross transformation that builds all pair
- * combinations of elements of both DataStreams, i.e., it builds a Cartesian
- * product.
- *
- * This method returns a StreamJoinOperator on which the
- * .onWindow(..) should be called to define the
- * window, and then the .where(..) and .equalTo(..) methods can be used to defin
- * the join keys.</p> The user can also use the apply method of the returned JoinedStream
- * to use custom join function.
- *
- */
- def cross[R](stream: DataStream[R]): StreamCrossOperator[T, R] =
- new StreamCrossOperator[T, R](javaStream, stream.getJavaStream)
-
- /**
- * Writes a DataStream to the standard output stream (stdout). For each
- * element of the DataStream the result of .toString is
- * written.
- *
- */
- def print(): DataStream[T] = javaStream.print()
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of .toString
- * is written.
- *
- */
- def writeAsText(path: String, millis: Long = 0): DataStream[T] =
- javaStream.writeAsText(path, millis)
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of .toString
- * is written.
- *
- */
- def writeAsCsv(path: String, millis: Long = 0): DataStream[T] =
- javaStream.writeAsCsv(path, millis)
-
- /**
- * Adds the given sink to this DataStream. Only streams with sinks added
- * will be executed once the StreamExecutionEnvironment.execute(...)
- * method is called.
- *
- */
- def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] =
- javaStream.addSink(sinkFuntion)
-
- /**
- * Adds the given sink to this DataStream. Only streams with sinks added
- * will be executed once the StreamExecutionEnvironment.execute(...)
- * method is called.
- *
- */
- def addSink(fun: T => Unit): DataStream[T] = {
- if (fun == null) {
- throw new NullPointerException("Sink function must not be null.")
- }
- val sinkFunction = new SinkFunction[T] {
- val cleanFun = clean(fun)
- def invoke(in: T) = cleanFun(in)
- }
- this.addSink(sinkFunction)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
deleted file mode 100644
index f61e34b..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/SplitDataStream.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
-import org.apache.flink.api.scala.streaming.StreamingConversions._
-
-/**
- * The SplitDataStream represents an operator that has been split using an
- * {@link OutputSelector}. Named outputs can be selected using the
- * {@link #select} function.
- *
- * @param <OUT>
- * The type of the output.
- */
-class SplitDataStream[T](javaStream: SplitJavaStream[T]) {
-
- /**
- * Gets the underlying java DataStream object.
- */
- private[flink] def getJavaStream: SplitJavaStream[T] = javaStream
-
- /**
- * Sets the output names for which the next operator will receive values.
- */
- def select(outputNames: String*): DataStream[T] = javaStream.select(outputNames: _*)
-
- /**
- * Selects all output names from a split data stream.
- */
- def selectAll(): DataStream[T] = javaStream.selectAll()
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
deleted file mode 100644
index cac2927..0000000
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamCrossOperator.scala
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.scala.streaming
-
-import org.apache.flink.api.common.functions.JoinFunction
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.common.typeutils.TypeSerializer
-import org.apache.flink.api.java.functions.KeySelector
-import org.apache.flink.api.scala.ClosureCleaner
-import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
-import org.apache.flink.streaming.api.datastream.TemporalOperator
-import scala.reflect.ClassTag
-import org.apache.commons.lang.Validate
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
-import org.apache.flink.streaming.api.function.co.CrossWindowFunction
-import org.apache.flink.api.common.functions.CrossFunction
-import org.apache.flink.api.scala.typeutils.CaseClassSerializer
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.api.scala.streaming.StreamExecutionEnvironment.clean
-import org.apache.flink.api.scala.streaming.StreamingConversions._
-
-class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
- TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
-
- override def createNextWindowOperator(): StreamCrossOperator.CrossWindow[I1, I2] = {
-
- val crossWindowFunction = StreamCrossOperator.getCrossWindowFunction(this,
- (l: I1, r: I2) => (l, r))
-
- val returnType = new CaseClassTypeInfo[(I1, I2)](
-
- classOf[(I1, I2)], Seq(input1.getType, input2.getType), Array("_1", "_2")) {
-
- override def createSerializer: TypeSerializer[(I1, I2)] = {
- val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
- for (i <- 0 until getArity) {
- fieldSerializers(i) = types(i).createSerializer
- }
-
- new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
- override def createInstance(fields: Array[AnyRef]) = {
- (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
- }
- }
- }
- }
-
- val javaStream = input1.connect(input2).addGeneralWindowCombine(
- crossWindowFunction,
- returnType, windowSize,
- slideInterval, timeStamp1, timeStamp2);
-
- new StreamCrossOperator.CrossWindow[I1, I2](this, javaStream)
- }
-}
-object StreamCrossOperator {
-
- private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2],
- javaStream: JavaStream[(I1, I2)]) extends
- DataStream[(I1, I2)](javaStream) {
-
- /**
- * Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf
- * call will be emitted.
- *
- */
- def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
-
- val invokable = new CoWindowInvokable[I1, I2, R](
- clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
- op.timeStamp2)
-
- javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
- invokable)
-
- javaStream.setType(implicitly[TypeInformation[R]])
- }
- }
-
- private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2],
- crossFunction: (I1, I2) => R):
- CrossWindowFunction[I1, I2, R] = {
- Validate.notNull(crossFunction, "Join function must not be null.")
-
- val crossFun = new CrossFunction[I1, I2, R] {
- val cleanFun = op.input1.clean(crossFunction)
-
- override def cross(first: I1, second: I2): R = {
- cleanFun(first, second)
- }
- }
-
- new CrossWindowFunction[I1, I2, R](crossFun)
- }
-
-}
[04/36] flink git commit: [scala] [streaming] Finished scala
StreamExecutionEnvrionment functionality + DataStream sinks + docs
Posted by mb...@apache.org.
[scala] [streaming] Finished scala StreamExecutionEnvrionment functionality + DataStream sinks + docs
Conflicts:
flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsCsvTest.java
flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsTextTest.java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d1b53b16
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d1b53b16
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d1b53b16
Branch: refs/heads/release-0.8
Commit: d1b53b16695231f54d912d847c11a406523c8d5b
Parents: 40efecf
Author: Gyula Fora <gy...@apache.org>
Authored: Sat Dec 13 01:08:08 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Jan 5 17:51:45 2015 +0100
----------------------------------------------------------------------
.../streaming/api/datastream/DataStream.java | 337 ++++---------------
.../environment/StreamExecutionEnvironment.java | 40 ++-
.../api/function/sink/WriteFormatAsCsv.java | 6 +-
.../api/function/sink/WriteSinkFunction.java | 15 +-
.../sink/WriteSinkFunctionByBatches.java | 49 ---
.../sink/WriteSinkFunctionByMillis.java | 7 +-
.../flink/streaming/api/WriteAsCsvTest.java | 176 ----------
.../flink/streaming/api/WriteAsTextTest.java | 177 ----------
.../flink/api/scala/streaming/DataStream.scala | 181 +++++++++-
.../streaming/StreamExecutionEnvironment.scala | 103 +++++-
10 files changed, 389 insertions(+), 702 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 1cf8d72..3a3cdc4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -44,9 +44,9 @@ import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
import org.apache.flink.streaming.api.function.sink.PrintSinkFunction;
import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.function.sink.WriteFormat;
import org.apache.flink.streaming.api.function.sink.WriteFormatAsCsv;
import org.apache.flink.streaming.api.function.sink.WriteFormatAsText;
-import org.apache.flink.streaming.api.function.sink.WriteSinkFunctionByBatches;
import org.apache.flink.streaming.api.function.sink.WriteSinkFunctionByMillis;
import org.apache.flink.streaming.api.invokable.SinkInvokable;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
@@ -275,7 +275,9 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output is
- * partitioned by the selected fields.
+ * partitioned by the selected fields. This setting only effects the how the
+ * outputs will be distributed between the parallel instances of the next
+ * processing operator.
*
* @param fields
* The fields to partition by.
@@ -289,7 +291,9 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output is
- * partitioned by the given field expressions.
+ * partitioned by the given field expressions. This setting only effects the
+ * how the outputs will be distributed between the parallel instances of the
+ * next processing operator.
*
* @param fields
* The fields expressions to partition by.
@@ -303,7 +307,9 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output is
- * partitioned using the given {@link KeySelector}.
+ * partitioned using the given {@link KeySelector}. This setting only
+ * effects the how the outputs will be distributed between the parallel
+ * instances of the next processing operator.
*
* @param keySelector
* @return
@@ -314,7 +320,9 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
- * are broadcasted to every parallel instance of the next component.
+ * are broadcasted to every parallel instance of the next component. This
+ * setting only effects the how the outputs will be distributed between the
+ * parallel instances of the next processing operator.
*
* @return The DataStream with broadcast partitioning set.
*/
@@ -324,7 +332,9 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
- * are shuffled to the next component.
+ * are shuffled to the next component. This setting only effects the how the
+ * outputs will be distributed between the parallel instances of the next
+ * processing operator.
*
* @return The DataStream with shuffle partitioning set.
*/
@@ -334,8 +344,10 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
- * are forwarded to the local subtask of the next component. This is the
- * default partitioner setting.
+ * are forwarded to the local subtask of the next component (whenever
+ * possible). This is the default partitioner setting. This setting only
+ * effects the how the outputs will be distributed between the parallel
+ * instances of the next processing operator.
*
* @return The DataStream with shuffle partitioning set.
*/
@@ -345,7 +357,9 @@ public class DataStream<OUT> {
/**
* Sets the partitioning of the {@link DataStream} so that the output tuples
- * are distributed evenly to the next component.
+ * are distributed evenly to the next component.This setting only effects
+ * the how the outputs will be distributed between the parallel instances of
+ * the next processing operator.
*
* @return The DataStream with shuffle partitioning set.
*/
@@ -547,9 +561,9 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the sum of the pojo data stream at
- * the given field expression. A field expression is either the name of a
- * public field or a getter method with parentheses of the
+ * Applies an aggregation that that gives the current sum of the pojo data
+ * stream at the given field expression. A field expression is either the
+ * name of a public field or a getter method with parentheses of the
* {@link DataStream}S underlying type. A dot can be used to drill down into
* objects, as in {@code "field1.getInnerField2()" }.
*
@@ -563,8 +577,8 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the minimum of the data stream at
- * the given position.
+ * Applies an aggregation that that gives the current minimum of the data
+ * stream at the given position.
*
* @param positionToMin
* The position in the data point to minimize
@@ -577,9 +591,9 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the minimum of the pojo data
- * stream at the given field expression. A field expression is either the
- * name of a public field or a getter method with parentheses of the
+ * Applies an aggregation that that gives the current minimum of the pojo
+ * data stream at the given field expression. A field expression is either
+ * the name of a public field or a getter method with parentheses of the
* {@link DataStream}S underlying type. A dot can be used to drill down into
* objects, as in {@code "field1.getInnerField2()" }.
*
@@ -594,8 +608,8 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that gives the maximum of the data stream at the
- * given position.
+ * Applies an aggregation that gives the current maximum of the data stream
+ * at the given position.
*
* @param positionToMax
* The position in the data point to maximize
@@ -608,9 +622,9 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the maximum of the pojo data
- * stream at the given field expression. A field expression is either the
- * name of a public field or a getter method with parentheses of the
+ * Applies an aggregation that that gives the current maximum of the pojo
+ * data stream at the given field expression. A field expression is either
+ * the name of a public field or a getter method with parentheses of the
* {@link DataStream}S underlying type. A dot can be used to drill down into
* objects, as in {@code "field1.getInnerField2()" }.
*
@@ -625,11 +639,11 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the minimum element of the pojo
- * data stream by the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
+ * Applies an aggregation that that gives the current minimum element of the
+ * pojo data stream by the given field expression. A field expression is
+ * either the name of a public field or a getter method with parentheses of
+ * the {@link DataStream}S underlying type. A dot can be used to drill down
+ * into objects, as in {@code "field1.getInnerField2()" }.
*
* @param field
* The field expression based on which the aggregation will be
@@ -645,11 +659,11 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the maximum element of the pojo
- * data stream by the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
+ * Applies an aggregation that that gives the current maximum element of the
+ * pojo data stream by the given field expression. A field expression is
+ * either the name of a public field or a getter method with parentheses of
+ * the {@link DataStream}S underlying type. A dot can be used to drill down
+ * into objects, as in {@code "field1.getInnerField2()" }.
*
* @param field
* The field expression based on which the aggregation will be
@@ -677,7 +691,7 @@ public class DataStream<OUT> {
public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
return this.minBy(positionToMinBy, true);
}
-
+
/**
* Applies an aggregation that that gives the current element with the
* minimum value at the given position, if more elements have the minimum
@@ -724,7 +738,7 @@ public class DataStream<OUT> {
public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
return this.maxBy(positionToMaxBy, true);
}
-
+
/**
* Applies an aggregation that that gives the current element with the
* maximum value at the given position, if more elements have the maximum
@@ -759,7 +773,8 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that gives the count of the values.
+ * Creates a new DataStream containing the current number (count) of
+ * received records.
*
* @return The transformed DataStream.
*/
@@ -826,9 +841,8 @@ public class DataStream<OUT> {
* @return The closed DataStream.
*/
public DataStreamSink<OUT> print() {
- DataStream<OUT> inputStream = this.copy();
PrintSinkFunction<OUT> printFunction = new PrintSinkFunction<OUT>();
- DataStreamSink<OUT> returnStream = addSink(inputStream, printFunction, getType());
+ DataStreamSink<OUT> returnStream = addSink(printFunction);
return returnStream;
}
@@ -841,28 +855,13 @@ public class DataStream<OUT> {
* @return The closed DataStream.
*/
public DataStreamSink<OUT> printToErr() {
- DataStream<OUT> inputStream = this.copy();
PrintSinkFunction<OUT> printFunction = new PrintSinkFunction<OUT>(true);
- DataStreamSink<OUT> returnStream = addSink(inputStream, printFunction, getType());
+ DataStreamSink<OUT> returnStream = addSink(printFunction);
return returnStream;
}
/**
- * Writes a DataStream to the file specified by path in text format. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written.
- *
- * @param path
- * is the path to the location where the tuples are written
- *
- * @return The closed DataStream
- */
- public DataStreamSink<OUT> writeAsText(String path) {
- return writeAsText(this, path, new WriteFormatAsText<OUT>(), 1, null);
- }
-
- /**
* Writes a DataStream to the file specified by path in text format. The
* writing is performed periodically, in every millis milliseconds. For
* every element of the DataStream the result of {@link Object#toString()}
@@ -876,122 +875,7 @@ public class DataStream<OUT> {
* @return The closed DataStream
*/
public DataStreamSink<OUT> writeAsText(String path, long millis) {
- return writeAsText(this, path, new WriteFormatAsText<OUT>(), millis, null);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically in equally sized batches. For every
- * element of the DataStream the result of {@link Object#toString()} is
- * written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param batchSize
- * is the size of the batches, i.e. the number of tuples written
- * to the file at a time
- *
- * @return The closed DataStream
- */
- public DataStreamSink<OUT> writeAsText(String path, int batchSize) {
- return writeAsText(this, path, new WriteFormatAsText<OUT>(), batchSize, null);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param millis
- * is the file update frequency
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * system time.
- *
- * @return The closed DataStream
- */
- public DataStreamSink<OUT> writeAsText(String path, long millis, OUT endTuple) {
- return writeAsText(this, path, new WriteFormatAsText<OUT>(), millis, endTuple);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically in equally sized batches. For every
- * element of the DataStream the result of {@link Object#toString()} is
- * written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param batchSize
- * is the size of the batches, i.e. the number of tuples written
- * to the file at a time
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * batchSize.
- *
- * @return The closed DataStream
- */
- public DataStreamSink<OUT> writeAsText(String path, int batchSize, OUT endTuple) {
- return writeAsText(this, path, new WriteFormatAsText<OUT>(), batchSize, endTuple);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
- * every element of the DataStream the result of {@link Object#toString()}
- * is written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param millis
- * is the file update frequency
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * system time.
- *
- * @return the data stream constructed
- */
- private DataStreamSink<OUT> writeAsText(DataStream<OUT> inputStream, String path,
- WriteFormatAsText<OUT> format, long millis, OUT endTuple) {
- DataStreamSink<OUT> returnStream = addSink(inputStream, new WriteSinkFunctionByMillis<OUT>(
- path, format, millis, endTuple), inputStream.typeInfo);
- return returnStream;
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically in equally sized batches. For every
- * element of the DataStream the result of {@link Object#toString()} is
- * written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param batchSize
- * is the size of the batches, i.e. the number of tuples written
- * to the file at a time
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * batchSize.
- *
- * @return the data stream constructed
- */
- private DataStreamSink<OUT> writeAsText(DataStream<OUT> inputStream, String path,
- WriteFormatAsText<OUT> format, int batchSize, OUT endTuple) {
- DataStreamSink<OUT> returnStream = addSink(inputStream,
- new WriteSinkFunctionByBatches<OUT>(path, format, batchSize, endTuple),
- inputStream.typeInfo);
- return returnStream;
+ return writeAsText(path, new WriteFormatAsText<OUT>(), millis);
}
/**
@@ -1004,8 +888,8 @@ public class DataStream<OUT> {
*
* @return The closed DataStream
*/
- public DataStreamSink<OUT> writeAsCsv(String path) {
- return writeAsCsv(this, path, new WriteFormatAsCsv<OUT>(), 1, null);
+ public DataStreamSink<OUT> writeAsText(String path) {
+ return writeAsText(path, 0);
}
/**
@@ -1022,74 +906,28 @@ public class DataStream<OUT> {
* @return The closed DataStream
*/
public DataStreamSink<OUT> writeAsCsv(String path, long millis) {
- return writeAsCsv(this, path, new WriteFormatAsCsv<OUT>(), millis, null);
- }
-
- /**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically in equally sized batches. For every
- * element of the DataStream the result of {@link Object#toString()} is
- * written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param batchSize
- * is the size of the batches, i.e. the number of tuples written
- * to the file at a time
- *
- * @return The closed DataStream
- */
- public DataStreamSink<OUT> writeAsCsv(String path, int batchSize) {
- return writeAsCsv(this, path, new WriteFormatAsCsv<OUT>(), batchSize, null);
+ if (!getType().isTupleType()) {
+ throw new RuntimeException("Only tuple data streams can be written in csv format");
+ }
+ return writeAsText(path, new WriteFormatAsCsv<OUT>(), millis);
}
/**
- * Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically, in every millis milliseconds. For
+ * Writes a DataStream to the file specified by path in text format. For
* every element of the DataStream the result of {@link Object#toString()}
* is written.
*
* @param path
* is the path to the location where the tuples are written
- * @param millis
- * is the file update frequency
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * system time.
*
* @return The closed DataStream
*/
- public DataStreamSink<OUT> writeAsCsv(String path, long millis, OUT endTuple) {
- return writeAsCsv(this, path, new WriteFormatAsCsv<OUT>(), millis, endTuple);
+ public DataStreamSink<OUT> writeAsCsv(String path) {
+ return writeAsCsv(path, 0);
}
/**
* Writes a DataStream to the file specified by path in text format. The
- * writing is performed periodically in equally sized batches. For every
- * element of the DataStream the result of {@link Object#toString()} is
- * written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param batchSize
- * is the size of the batches, i.e. the number of tuples written
- * to the file at a time
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * batchSize.
- *
- * @return The closed DataStream
- */
- public DataStreamSink<OUT> writeAsCsv(String path, int batchSize, OUT endTuple) {
- return writeAsCsv(this, path, new WriteFormatAsCsv<OUT>(), batchSize, endTuple);
- }
-
- /**
- * Writes a DataStream to the file specified by path in csv format. The
* writing is performed periodically, in every millis milliseconds. For
* every element of the DataStream the result of {@link Object#toString()}
* is written.
@@ -1098,45 +936,12 @@ public class DataStream<OUT> {
* is the path to the location where the tuples are written
* @param millis
* is the file update frequency
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * system time.
- *
- * @return the data stream constructed
- */
- private DataStreamSink<OUT> writeAsCsv(DataStream<OUT> inputStream, String path,
- WriteFormatAsCsv<OUT> format, long millis, OUT endTuple) {
- DataStreamSink<OUT> returnStream = addSink(inputStream, new WriteSinkFunctionByMillis<OUT>(
- path, format, millis, endTuple), inputStream.typeInfo);
- return returnStream;
- }
-
- /**
- * Writes a DataStream to the file specified by path in csv format. The
- * writing is performed periodically in equally sized batches. For every
- * element of the DataStream the result of {@link Object#toString()} is
- * written.
- *
- * @param path
- * is the path to the location where the tuples are written
- * @param batchSize
- * is the size of the batches, i.e. the number of tuples written
- * to the file at a time
- * @param endTuple
- * is a special tuple indicating the end of the stream. If an
- * endTuple is caught, the last pending batch of tuples will be
- * immediately appended to the target file regardless of the
- * batchSize.
*
* @return the data stream constructed
*/
- private DataStreamSink<OUT> writeAsCsv(DataStream<OUT> inputStream, String path,
- WriteFormatAsCsv<OUT> format, int batchSize, OUT endTuple) {
- DataStreamSink<OUT> returnStream = addSink(inputStream,
- new WriteSinkFunctionByBatches<OUT>(path, format, batchSize, endTuple),
- inputStream.typeInfo);
+ private DataStreamSink<OUT> writeAsText(String path, WriteFormat<OUT> format, long millis) {
+ DataStreamSink<OUT> returnStream = addSink(new WriteSinkFunctionByMillis<OUT>(path, format,
+ millis));
return returnStream;
}
@@ -1241,21 +1046,13 @@ public class DataStream<OUT> {
* @return The closed DataStream.
*/
public DataStreamSink<OUT> addSink(SinkFunction<OUT> sinkFunction) {
- return addSink(this.copy(), sinkFunction);
- }
-
- private DataStreamSink<OUT> addSink(DataStream<OUT> inputStream, SinkFunction<OUT> sinkFunction) {
- return addSink(inputStream, sinkFunction, getType());
- }
- private DataStreamSink<OUT> addSink(DataStream<OUT> inputStream,
- SinkFunction<OUT> sinkFunction, TypeInformation<OUT> inTypeInfo) {
DataStreamSink<OUT> returnStream = new DataStreamSink<OUT>(environment, "sink", typeInfo);
jobGraphBuilder.addStreamVertex(returnStream.getId(), new SinkInvokable<OUT>(
- clean(sinkFunction)), inTypeInfo, null, "sink", degreeOfParallelism);
+ clean(sinkFunction)), getType(), null, "sink", degreeOfParallelism);
- inputStream.connectGraph(inputStream.copy(), returnStream.getId(), 0);
+ this.connectGraph(this.copy(), returnStream.getId(), 0);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index e9581a1..5bf2164 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -212,7 +212,8 @@ public abstract class StreamExecutionEnvironment {
/**
* Creates a DataStream that represents the Strings produced by reading the
* given file line wise multiple times(infinite). The file will be read with
- * the system's default character set.
+ * the system's default character set. This functionality can be used for
+ * testing a topology.
*
* @param filePath
* The path of the file, as a URI (e.g.,
@@ -350,8 +351,17 @@ public abstract class StreamExecutionEnvironment {
return addSource(new GenSequenceFunction(from, to));
}
+ private DataStreamSource<String> addFileSource(InputFormat<String, ?> inputFormat,
+ TypeInformation<String> typeInfo) {
+ FileSourceFunction function = new FileSourceFunction(inputFormat, typeInfo);
+ DataStreamSource<String> returnStream = addSource(function);
+ jobGraphBuilder.setInputFormat(returnStream.getId(), inputFormat);
+ return returnStream;
+ }
+
/**
- * Ads a data source thus opening a {@link DataStream}.
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality.
*
* @param function
* the user defined function
@@ -371,11 +381,27 @@ public abstract class StreamExecutionEnvironment {
return returnStream;
}
- private DataStreamSource<String> addFileSource(InputFormat<String, ?> inputFormat,
- TypeInformation<String> typeInfo) {
- FileSourceFunction function = new FileSourceFunction(inputFormat, typeInfo);
- DataStreamSource<String> returnStream = addSource(function);
- jobGraphBuilder.setInputFormat(returnStream.getId(), inputFormat);
+ /**
+ * Ads a data source with a custom type information thus opening a
+ * {@link DataStream}. Only in very special cases does the user need to
+ * support type information. Otherwise use
+ * {@link #addSource(SourceFunction)}
+ *
+ * @param function
+ * the user defined function
+ * @param outTypeInfo
+ * the user defined type information for the stream
+ * @param <OUT>
+ * type of the returned stream
+ * @return the data stream constructed
+ */
+ public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function,
+ TypeInformation<OUT> outTypeInfo) {
+
+ DataStreamSource<OUT> returnStream = new DataStreamSource<OUT>(this, "source", outTypeInfo);
+
+ jobGraphBuilder.addStreamVertex(returnStream.getId(), new SourceInvokable<OUT>(function),
+ null, outTypeInfo, "source", 1);
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
index ebc38bb..b22fd80 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteFormatAsCsv.java
@@ -25,7 +25,7 @@ import java.util.ArrayList;
/**
* Writes tuples in csv format.
- *
+ *
* @param <IN>
* Input tuple type
*/
@@ -37,8 +37,8 @@ public class WriteFormatAsCsv<IN> extends WriteFormat<IN> {
try {
PrintWriter outStream = new PrintWriter(new BufferedWriter(new FileWriter(path, true)));
for (IN tupleToWrite : tupleList) {
- outStream.println(tupleToWrite.toString().substring(1,
- tupleToWrite.toString().length() - 1));
+ String strTuple = tupleToWrite.toString();
+ outStream.println(strTuple.substring(1, strTuple.length() - 1));
}
outStream.close();
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
index 01298b0..0c52afc 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunction.java
@@ -35,13 +35,11 @@ public abstract class WriteSinkFunction<IN> implements SinkFunction<IN> {
protected final String path;
protected ArrayList<IN> tupleList = new ArrayList<IN>();
- protected final IN endTuple;
protected WriteFormat<IN> format;
- public WriteSinkFunction(String path, WriteFormat<IN> format, IN endTuple) {
+ public WriteSinkFunction(String path, WriteFormat<IN> format) {
this.path = path;
this.format = format;
- this.endTuple = endTuple;
cleanFile(path);
}
@@ -82,16 +80,13 @@ public abstract class WriteSinkFunction<IN> implements SinkFunction<IN> {
*/
@Override
public void invoke(IN tuple) {
- if (!tuple.equals(endTuple)) {
- tupleList.add(tuple);
- if (updateCondition()) {
- format.write(path, tupleList);
- resetParameters();
- }
- } else {
+
+ tupleList.add(tuple);
+ if (updateCondition()) {
format.write(path, tupleList);
resetParameters();
}
+
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByBatches.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByBatches.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByBatches.java
deleted file mode 100644
index 5012e25..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByBatches.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.sink;
-
-
-/**
- * Implementation of WriteSinkFunction. Writes tuples to file in equally sized
- * batches.
- *
- * @param <IN>
- * Input tuple type
- */
-public class WriteSinkFunctionByBatches<IN> extends WriteSinkFunction<IN> {
- private static final long serialVersionUID = 1L;
-
- private final int batchSize;
-
- public WriteSinkFunctionByBatches(String path, WriteFormat<IN> format, int batchSize,
- IN endTuple) {
- super(path, format, endTuple);
- this.batchSize = batchSize;
- }
-
- @Override
- protected boolean updateCondition() {
- return tupleList.size() >= batchSize;
- }
-
- @Override
- protected void resetParameters() {
- tupleList.clear();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
index 1fa978e..ee6df94 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/sink/WriteSinkFunctionByMillis.java
@@ -17,11 +17,10 @@
package org.apache.flink.streaming.api.function.sink;
-
/**
* Implementation of WriteSinkFunction. Writes tuples to file in every millis
* milliseconds.
- *
+ *
* @param <IN>
* Input tuple type
*/
@@ -31,8 +30,8 @@ public class WriteSinkFunctionByMillis<IN> extends WriteSinkFunction<IN> {
private final long millis;
private long lastTime;
- public WriteSinkFunctionByMillis(String path, WriteFormat<IN> format, long millis, IN endTuple) {
- super(path, format, endTuple);
+ public WriteSinkFunctionByMillis(String path, WriteFormat<IN> format, long millis) {
+ super(path, format);
this.millis = millis;
lastTime = System.currentTimeMillis();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsCsvTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsCsvTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsCsvTest.java
deleted file mode 100644
index 6b1dd5a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsCsvTest.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertTrue;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.util.Collector;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class WriteAsCsvTest {
-
- private static final String PREFIX = System.getProperty("java.io.tmpdir") + "/" + WriteAsCsvTest.class.getSimpleName() + "_";
-
- private static final long MEMORYSIZE = 32;
-
- private static List<String> result1 = new ArrayList<String>();
- private static List<String> result2 = new ArrayList<String>();
- private static List<String> result3 = new ArrayList<String>();
- private static List<String> result4 = new ArrayList<String>();
- private static List<String> result5 = new ArrayList<String>();
-
- private static List<String> expected1 = new ArrayList<String>();
- private static List<String> expected2 = new ArrayList<String>();
- private static List<String> expected3 = new ArrayList<String>();
- private static List<String> expected4 = new ArrayList<String>();
- private static List<String> expected5 = new ArrayList<String>();
-
- public static final class MySource1 implements SourceFunction<Tuple1<Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Collector<Tuple1<Integer>> collector) throws Exception {
- for (int i = 0; i < 27; i++) {
- collector.collect(new Tuple1<Integer>(i));
- }
- }
- }
-
- private static void readFile(String path, List<String> result) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(path));
- String line;
- line = br.readLine();
- while (line != null) {
- result.add(line);
- line = br.readLine();
- }
- br.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private static void fillExpected1() {
- for (int i = 0; i < 27; i++) {
- expected1.add(i + "");
- }
- }
-
- private static void fillExpected2() {
- for (int i = 0; i < 25; i++) {
- expected2.add(i + "");
- }
- }
-
- private static void fillExpected3() {
- for (int i = 0; i < 20; i++) {
- expected3.add(i + "");
- }
- }
-
- private static void fillExpected4() {
- for (int i = 0; i < 26; i++) {
- expected4.add(i + "");
- }
- }
-
- private static void fillExpected5() {
- for (int i = 0; i < 14; i++) {
- expected5.add(i + "");
- }
-
- for (int i = 15; i < 25; i++) {
- expected5.add(i + "");
- }
- }
-
- @BeforeClass
- public static void createFileCleanup() {
- Runnable r = new Runnable() {
-
- @Override
- public void run() {
- try { new File(PREFIX + "test1.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test2.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test3.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test4.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test5.txt").delete(); } catch (Throwable t) {}
- }
- };
-
- Runtime.getRuntime().addShutdownHook(new Thread(r));
- }
-
- @Test
- public void test() throws Exception {
- LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream1 = env.addSource(new MySource1()).writeAsCsv(PREFIX + "test1.txt");
-
- fillExpected1();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream2 = env.addSource(new MySource1()).writeAsCsv(PREFIX + "test2.txt", 5);
-
- fillExpected2();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream3 = env.addSource(new MySource1()).writeAsCsv(PREFIX + "test3.txt", 10);
-
- fillExpected3();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream4 = env.addSource(new MySource1()).writeAsCsv(PREFIX + "test4.txt", 10, new Tuple1<Integer>(26));
-
- fillExpected4();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream5 = env.addSource(new MySource1()).writeAsCsv(PREFIX + "test5.txt", 10, new Tuple1<Integer>(14));
-
- fillExpected5();
-
- env.executeTest(MEMORYSIZE);
-
- readFile(PREFIX + "test1.txt", result1);
- readFile(PREFIX + "test2.txt", result2);
- readFile(PREFIX + "test3.txt", result3);
- readFile(PREFIX + "test4.txt", result4);
- readFile(PREFIX + "test5.txt", result5);
-
- assertTrue(expected1.equals(result1));
- assertTrue(expected2.equals(result2));
- assertTrue(expected3.equals(result3));
- assertTrue(expected4.equals(result4));
- assertTrue(expected5.equals(result5));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsTextTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsTextTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsTextTest.java
deleted file mode 100644
index e21f21d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WriteAsTextTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.util.Collector;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class WriteAsTextTest {
-
- private static final String PREFIX = System.getProperty("java.io.tmpdir") + "/" + WriteAsTextTest.class.getSimpleName() + "_";
-
- private static final long MEMORYSIZE = 32;
-
- private static List<String> result1 = new ArrayList<String>();
- private static List<String> result2 = new ArrayList<String>();
- private static List<String> result3 = new ArrayList<String>();
- private static List<String> result4 = new ArrayList<String>();
- private static List<String> result5 = new ArrayList<String>();
-
- private static List<String> expected1 = new ArrayList<String>();
- private static List<String> expected2 = new ArrayList<String>();
- private static List<String> expected3 = new ArrayList<String>();
- private static List<String> expected4 = new ArrayList<String>();
- private static List<String> expected5 = new ArrayList<String>();
-
- public static final class MySource1 implements SourceFunction<Tuple1<Integer>> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void invoke(Collector<Tuple1<Integer>> collector) throws Exception {
- for (int i = 0; i < 27; i++) {
- collector.collect(new Tuple1<Integer>(i));
- }
- }
- }
-
- private static void readFile(String path, List<String> result) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(path));
- String line;
- line = br.readLine();
- while (line != null) {
- result.add(line);
- line = br.readLine();
- }
- br.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private static void fillExpected1() {
- for (int i = 0; i < 27; i++) {
- expected1.add("(" + i + ")");
- }
- }
-
- private static void fillExpected2() {
- for (int i = 0; i < 25; i++) {
- expected2.add("(" + i + ")");
- }
- }
-
- private static void fillExpected3() {
- for (int i = 0; i < 20; i++) {
- expected3.add("(" + i + ")");
- }
- }
-
- private static void fillExpected4() {
- for (int i = 0; i < 26; i++) {
- expected4.add("(" + i + ")");
- }
- }
-
- private static void fillExpected5() {
- for (int i = 0; i < 14; i++) {
- expected5.add("(" + i + ")");
- }
-
- for (int i = 15; i < 25; i++) {
- expected5.add("(" + i + ")");
- }
- }
-
- @BeforeClass
- public static void createFileCleanup() {
- Runnable r = new Runnable() {
-
- @Override
- public void run() {
- try { new File(PREFIX + "test1.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test2.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test3.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test4.txt").delete(); } catch (Throwable t) {}
- try { new File(PREFIX + "test5.txt").delete(); } catch (Throwable t) {}
- }
- };
-
- Runtime.getRuntime().addShutdownHook(new Thread(r));
- }
-
- @Test
- public void test() throws Exception {
- LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream1 = env.addSource(new MySource1()).writeAsText(PREFIX + "test1.txt");
-
- fillExpected1();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream2 = env.addSource(new MySource1()).writeAsText(PREFIX + "test2.txt", 5);
-
- fillExpected2();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream3 = env.addSource(new MySource1()).writeAsText(PREFIX + "test3.txt", 10);
-
- fillExpected3();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream4 = env.addSource(new MySource1()).writeAsText(PREFIX + "test4.txt", 10, new Tuple1<Integer>(26));
-
- fillExpected4();
-
- @SuppressWarnings("unused")
- DataStream<Tuple1<Integer>> dataStream5 = env.addSource(new MySource1()).writeAsText(PREFIX + "test5.txt", 10, new Tuple1<Integer>(14));
-
- fillExpected5();
-
- env.executeTest(MEMORYSIZE);
-
- readFile(PREFIX + "test1.txt", result1);
- readFile(PREFIX + "test2.txt", result2);
- readFile(PREFIX + "test3.txt", result3);
- readFile(PREFIX + "test4.txt", result4);
- readFile(PREFIX + "test5.txt", result5);
-
- assertEquals(expected1,result1);
- assertEquals(expected2,result2);
- assertEquals(expected3,result3);
- assertEquals(expected4,result4);
- assertEquals(expected5,result5);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
index b10bdc6..e96f5eb 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/DataStream.scala
@@ -37,6 +37,7 @@ import org.apache.flink.streaming.api.datastream.GroupedDataStream
import org.apache.flink.api.common.functions.ReduceFunction
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.common.functions.FilterFunction
+import org.apache.flink.streaming.api.function.sink.SinkFunction
class DataStream[T](javaStream: JavaStream[T]) {
@@ -86,15 +87,36 @@ class DataStream[T](javaStream: JavaStream[T]) {
"parallelism.")
}
+ /**
+ * Creates a new DataStream by merging DataStream outputs of
+ * the same type with each other. The DataStreams merged using this operator
+ * will be transformed simultaneously.
+ *
+ */
def merge(dataStreams: DataStream[T]*): DataStream[T] =
new DataStream[T](javaStream.merge(dataStreams.map(_.getJavaStream): _*))
+ /**
+ * Groups the elements of a DataStream by the given key positions (for tuple/array types) to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
def groupBy(fields: Int*): DataStream[T] =
new DataStream[T](javaStream.groupBy(fields: _*))
+ /**
+ * Groups the elements of a DataStream by the given field expressions to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
def groupBy(firstField: String, otherFields: String*): DataStream[T] =
new DataStream[T](javaStream.groupBy(firstField +: otherFields.toArray: _*))
+ /**
+ * Groups the elements of a DataStream by the given K key to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
def groupBy[K: TypeInformation](fun: T => K): DataStream[T] = {
val keyExtractor = new KeySelector[T, K] {
@@ -104,12 +126,27 @@ class DataStream[T](javaStream: JavaStream[T]) {
new DataStream[T](javaStream.groupBy(keyExtractor))
}
+ /**
+ * Sets the partitioning of the DataStream so that the output is
+ * partitioned by the selected fields. This setting only effects the how the outputs will be distributed between the parallel instances of the next processing operator.
+ *
+ */
def partitionBy(fields: Int*): DataStream[T] =
new DataStream[T](javaStream.partitionBy(fields: _*))
+ /**
+ * Sets the partitioning of the DataStream so that the output is
+ * partitioned by the selected fields. This setting only effects the how the outputs will be distributed between the parallel instances of the next processing operator.
+ *
+ */
def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
new DataStream[T](javaStream.partitionBy(firstField +: otherFields.toArray: _*))
+ /**
+ * Sets the partitioning of the DataStream so that the output is
+ * partitioned by the given Key. This setting only effects the how the outputs will be distributed between the parallel instances of the next processing operator.
+ *
+ */
def partitionBy[K: TypeInformation](fun: T => K): DataStream[T] = {
val keyExtractor = new KeySelector[T, K] {
@@ -119,56 +156,124 @@ class DataStream[T](javaStream: JavaStream[T]) {
new DataStream[T](javaStream.partitionBy(keyExtractor))
}
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are broadcasted to every parallel instance of the next component. This
+ * setting only effects the how the outputs will be distributed between the
+ * parallel instances of the next processing operator.
+ *
+ */
def broadcast: DataStream[T] = new DataStream[T](javaStream.broadcast())
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are shuffled to the next component. This setting only effects the how the
+ * outputs will be distributed between the parallel instances of the next
+ * processing operator.
+ *
+ */
def shuffle: DataStream[T] = new DataStream[T](javaStream.shuffle())
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are forwarded to the local subtask of the next component (whenever
+ * possible). This is the default partitioner setting. This setting only
+ * effects the how the outputs will be distributed between the parallel
+ * instances of the next processing operator.
+ *
+ */
def forward: DataStream[T] = new DataStream[T](javaStream.forward())
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are distributed evenly to the next component.This setting only effects
+ * the how the outputs will be distributed between the parallel instances of
+ * the next processing operator.
+ *
+ */
def distribute: DataStream[T] = new DataStream[T](javaStream.distribute())
+ /**
+ * Applies an aggregation that that gives the current maximum of the data stream at
+ * the given position.
+ *
+ */
def max(field: Any): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.max(field))
case field: String => return new DataStream[T](javaStream.max(field))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
}
+ /**
+ * Applies an aggregation that that gives the current minimum of the data stream at
+ * the given position.
+ *
+ */
def min(field: Any): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.min(field))
case field: String => return new DataStream[T](javaStream.min(field))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
}
+ /**
+ * Applies an aggregation that sums the data stream at the given position.
+ *
+ */
def sum(field: Any): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.sum(field))
case field: String => return new DataStream[T](javaStream.sum(field))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
}
+ /**
+ * Applies an aggregation that that gives the current maximum element of the data stream by
+ * the given position. When equality, returns the first.
+ *
+ */
def maxBy(field: Any): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.maxBy(field))
case field: String => return new DataStream[T](javaStream.maxBy(field))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
}
+ /**
+ * Applies an aggregation that that gives the current minimum element of the data stream by
+ * the given position. When equality, returns the first.
+ *
+ */
def minBy(field: Any): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.minBy(field))
case field: String => return new DataStream[T](javaStream.minBy(field))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
}
+ /**
+ * Applies an aggregation that that gives the current minimum element of the data stream by
+ * the given position. When equality, the user can set to get the first or last element with the minimal value.
+ *
+ */
def minBy(field: Any, first: Boolean): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.minBy(field, first))
case field: String => return new DataStream[T](javaStream.minBy(field, first))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
}
+ /**
+ * Applies an aggregation that that gives the current maximum element of the data stream by
+ * the given position. When equality, the user can set to get the first or last element with the maximal value.
+ *
+ */
def maxBy(field: Any, first: Boolean): DataStream[T] = field match {
case field: Int => return new DataStream[T](javaStream.maxBy(field, first))
case field: String => return new DataStream[T](javaStream.maxBy(field, first))
case _ => throw new IllegalArgumentException("Aggregations are only supported by field position (Int) or field expression (String)")
}
+ /**
+ * Creates a new DataStream containing the current number (count) of
+ * received records.
+ *
+ */
def count: DataStream[java.lang.Long] = new DataStream[java.lang.Long](javaStream.count())
/**
@@ -239,7 +344,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
}
/**
- * Creates a new [[DataStream]] by merging the elements of this DataStream using an associative reduce
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream using an associative reduce
* function.
*/
def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
@@ -253,7 +358,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
}
/**
- * Creates a new [[DataStream]] by merging the elements of this DataStream using an associative reduce
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream using an associative reduce
* function.
*/
def reduce(fun: (T, T) => T): DataStream[T] = {
@@ -268,7 +373,7 @@ class DataStream[T](javaStream: JavaStream[T]) {
}
/**
- * Creates a new DataSet that contains only the elements satisfying the given filter predicate.
+ * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
*/
def filter(filter: FilterFunction[T]): DataStream[T] = {
if (filter == null) {
@@ -277,6 +382,9 @@ class DataStream[T](javaStream: JavaStream[T]) {
new DataStream[T](javaStream.filter(filter))
}
+ /**
+ * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
+ */
def filter(fun: T => Boolean): DataStream[T] = {
if (fun == null) {
throw new NullPointerException("Filter function must not be null.")
@@ -288,6 +396,71 @@ class DataStream[T](javaStream: JavaStream[T]) {
this.filter(filter)
}
- def print() = javaStream.print()
+ /**
+ * Writes a DataStream to the standard output stream (stdout). For each
+ * element of the DataStream the result of .toString is
+ * written.
+ *
+ */
+ def print(): DataStream[T] = new DataStream[T](javaStream.print())
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsText(path: String, millis: Long): DataStream[T] = new DataStream[T](javaStream.writeAsText(path, millis))
+
+ /**
+ * Writes a DataStream to the file specified by path in text format.
+ * For every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsText(path: String): DataStream[T] = new DataStream[T](javaStream.writeAsText(path))
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsCsv(path: String, millis: Long): DataStream[T] = new DataStream[T](javaStream.writeAsCsv(path, millis))
+
+ /**
+ * Writes a DataStream to the file specified by path in text format.
+ * For every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsCsv(path: String): DataStream[T] = new DataStream[T](javaStream.writeAsCsv(path))
+
+ /**
+ * Adds the given sink to this DataStream. Only streams with sinks added
+ * will be executed once the StreamExecutionEnvironment.execute(...)
+ * method is called.
+ *
+ */
+ def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] = new DataStream[T](javaStream.addSink(sinkFuntion))
+
+ /**
+ * Adds the given sink to this DataStream. Only streams with sinks added
+ * will be executed once the StreamExecutionEnvironment.execute(...)
+ * method is called.
+ *
+ */
+ def addSink(fun: T => Unit): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Sink function must not be null.")
+ }
+ val sinkFunction = new SinkFunction[T] {
+ val cleanFun = clean(fun)
+ def invoke(in: T) = cleanFun(in)
+ }
+ this.addSink(sinkFunction)
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/d1b53b16/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
index df6c561..e4a7b48 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/streaming/StreamExecutionEnvironment.scala
@@ -25,6 +25,7 @@ import scala.reflect.ClassTag
import org.apache.flink.streaming.api.datastream.DataStreamSource
import org.apache.flink.streaming.api.invokable.SourceInvokable
import org.apache.flink.streaming.api.function.source.FromElementsFunction
+import org.apache.flink.streaming.api.function.source.SourceFunction
class StreamExecutionEnvironment(javaEnv: JavaEnv) {
@@ -44,10 +45,78 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
*/
def getDegreeOfParallelism = javaEnv.getDegreeOfParallelism
+ /**
+ * Sets the maximum time frequency (milliseconds) for the flushing of the
+ * output buffers. By default the output buffers flush frequently to provide
+ * low latency and to aid smooth developer experience. Setting the parameter
+ * can result in three logical modes:
+ *
+ * <ul>
+ * <li>
+ * A positive integer triggers flushing periodically by that integer</li>
+ * <li>
+ * 0 triggers flushing after every record thus minimizing latency</li>
+ * <li>
+ * -1 triggers flushing only when the output buffer is full thus maximizing
+ * throughput</li>
+ * </ul>
+ *
+ */
+ def setBufferTimeout(timeoutMillis: Long): StreamExecutionEnvironment = {
+ javaEnv.setBufferTimeout(timeoutMillis)
+ this
+ }
+
+ /**
+ * Gets the default buffer timeout set for this environment
+ */
+ def getBufferTimout: Long = javaEnv.getBufferTimeout()
+
+ /**
+ * Creates a DataStream that represents the Strings produced by reading the
+ * given file line wise. The file will be read with the system's default
+ * character set.
+ *
+ */
+ def readTextFile(filePath: String): DataStream[String] =
+ new DataStream[String](javaEnv.readTextFile(filePath))
+
+ /**
+ * Creates a DataStream that represents the Strings produced by reading the
+ * given file line wise multiple times(infinite). The file will be read with
+ * the system's default character set. This functionality can be used for
+ * testing a topology.
+ *
+ */
+ def readTextStream(StreamPath: String): DataStream[String] =
+ new DataStream[String](javaEnv.readTextStream(StreamPath))
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set.
+ *
+ */
+ def socketTextStream(hostname: String, port: Int, delimiter: Char): DataStream[String] =
+ new DataStream[String](javaEnv.socketTextStream(hostname, port, delimiter))
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set, uses '\n' as delimiter.
+ *
+ */
+ def socketTextStream(hostname: String, port: Int): DataStream[String] =
+ new DataStream[String](javaEnv.socketTextStream(hostname, port))
+
+ /**
+ * Creates a new DataStream that contains a sequence of numbers.
+ *
+ */
def generateSequence(from: Long, to: Long): DataStream[java.lang.Long] = new DataStream(javaEnv.generateSequence(from, to))
/**
- * Creates a new data stream that contains the given elements. The elements must all be of the
+ * Creates a DataStream that contains the given elements. The elements must all be of the
* same type and must be serializable.
*
* * Note that this operation will result in a non-parallel data source, i.e. a data source with
@@ -78,8 +147,38 @@ class StreamExecutionEnvironment(javaEnv: JavaEnv) {
new DataStream(returnStream)
}
+ /**
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality.
+ *
+ */
+ def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
+ Validate.notNull(function, "Function must not be null.")
+ val typeInfo = implicitly[TypeInformation[T]]
+ new DataStream[T](javaEnv.addSource(function, typeInfo))
+ }
+
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with a generated
+ * default name.
+ *
+ */
def execute() = javaEnv.execute()
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with the provided name
+ *
+ */
+ def execute(jobName: String) = javaEnv.execute(jobName)
+
}
object StreamExecutionEnvironment {
@@ -108,7 +207,7 @@ object StreamExecutionEnvironment {
* Creates a remote execution environment. The remote environment sends (parts of) the program to
* a cluster for execution. Note that all file paths used in the program must be accessible from
* the cluster. The execution will use the cluster's default degree of parallelism, unless the
- * parallelism is set explicitly via [[ExecutionEnvironment.setDegreeOfParallelism()]].
+ * parallelism is set explicitly via [[StreamExecutionEnvironment.setDegreeOfParallelism()]].
*
* @param host The host name or address of the master (JobManager),
* where the program should be executed.
[29/36] flink git commit: [streaming] Temporal operator windowing
syntax update
Posted by mb...@apache.org.
[streaming] Temporal operator windowing syntax update
Conflicts:
flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fb86fde2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fb86fde2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fb86fde2
Branch: refs/heads/release-0.8
Commit: fb86fde2b94d53e133efa542520a48d220e168a9
Parents: e3700c3
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Jan 5 15:24:59 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 00:22:57 2015 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 4 +
.../streaming/api/datastream/DataStream.java | 4 +-
.../api/datastream/DataStreamSink.java | 2 +-
.../api/datastream/GroupedDataStream.java | 2 +-
.../api/datastream/IterativeDataStream.java | 2 +-
.../datastream/SingleOutputStreamOperator.java | 2 +-
.../api/datastream/StreamCrossOperator.java | 86 ------
.../api/datastream/StreamJoinOperator.java | 257 ------------------
.../api/datastream/TemporalOperator.java | 102 -------
.../temporaloperator/StreamCrossOperator.java | 101 +++++++
.../temporaloperator/StreamJoinOperator.java | 272 +++++++++++++++++++
.../temporaloperator/TemporalOperator.java | 124 +++++++++
.../temporaloperator/TemporalWindow.java | 45 +++
.../operator/co/CoWindowInvokable.java | 4 +
.../streaming/api/windowing/helper/Time.java | 20 +-
.../streaming/api/WindowCrossJoinTest.java | 21 +-
.../streaming/examples/join/WindowJoin.java | 3 +-
.../scala/examples/windowing/WindowJoin.scala | 7 +-
.../api/scala/StreamCrossOperator.scala | 17 +-
.../api/scala/StreamJoinOperator.scala | 47 ++--
.../streaming/api/scala/TemporalOperator.scala | 43 +++
21 files changed, 662 insertions(+), 503 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index c826274..441360c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -509,6 +509,10 @@ public class JobGraphBuilder {
invokableObjects.put(id, invokableObject);
}
+ public StreamInvokable<?, ?> getInvokable(String id) {
+ return invokableObjects.get(id);
+ }
+
public <OUT> void setOutType(String id, TypeInformation<OUT> outType) {
StreamRecordSerializer<OUT> serializer = new StreamRecordSerializer<OUT>(outType);
typeSerializersOut1.put(id, serializer);
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 8d0020e..e969647 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -38,6 +38,8 @@ import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.streaming.api.JobGraphBuilder;
+import org.apache.flink.streaming.api.datastream.temporaloperator.StreamCrossOperator;
+import org.apache.flink.streaming.api.datastream.temporaloperator.StreamJoinOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
@@ -1180,7 +1182,7 @@ public class DataStream<OUT> {
*
* @return The copy
*/
- protected DataStream<OUT> copy() {
+ public DataStream<OUT> copy() {
return new DataStream<OUT>(this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
index 369c3eb..61fc557 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStreamSink.java
@@ -38,7 +38,7 @@ public class DataStreamSink<IN> extends SingleOutputStreamOperator<IN, DataStrea
}
@Override
- protected DataStreamSink<IN> copy() {
+ public DataStreamSink<IN> copy() {
throw new RuntimeException("Data stream sinks cannot be copied");
}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
index 160ef8d..4a5c0c2 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/GroupedDataStream.java
@@ -199,7 +199,7 @@ public class GroupedDataStream<OUT> extends DataStream<OUT> {
}
@Override
- protected GroupedDataStream<OUT> copy() {
+ public GroupedDataStream<OUT> copy() {
return new GroupedDataStream<OUT>(this);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index 78518c0..d7467d1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -72,7 +72,7 @@ public class IterativeDataStream<IN> extends
}
@Override
- protected IterativeDataStream<IN> copy() {
+ public IterativeDataStream<IN> copy() {
return new IterativeDataStream<IN>(this, iterationID, waitTime);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index c19517b..4b6edc0 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -186,7 +186,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
}
@Override
- protected SingleOutputStreamOperator<OUT, O> copy() {
+ public SingleOutputStreamOperator<OUT, O> copy() {
return new SingleOutputStreamOperator<OUT, O>(this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
deleted file mode 100644
index 2dd7bc0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.functions.CrossFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.operators.CrossOperator;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-
-public class StreamCrossOperator<I1, I2> extends
- TemporalOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
-
- public StreamCrossOperator(DataStream<I1> input1, DataStream<I2> input2) {
- super(input1, input2);
- }
-
- @Override
- protected CrossWindow<I1, I2> createNextWindowOperator() {
-
- CrossWindowFunction<I1, I2, Tuple2<I1, I2>> crossWindowFunction = new CrossWindowFunction<I1, I2, Tuple2<I1, I2>>(
- input1.clean(new CrossOperator.DefaultCrossFunction<I1, I2>()));
-
- return new CrossWindow<I1, I2>(this, input1.connect(input2).addGeneralWindowCombine(
- crossWindowFunction,
- new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType()), windowSize,
- slideInterval, timeStamp1, timeStamp2));
- }
-
- public static class CrossWindow<I1, I2> extends
- SingleOutputStreamOperator<Tuple2<I1, I2>, CrossWindow<I1, I2>> {
-
- private StreamCrossOperator<I1, I2> op;
-
- public CrossWindow(StreamCrossOperator<I1, I2> op, DataStream<Tuple2<I1, I2>> ds) {
- super(ds);
- this.op = op;
- }
-
- /**
- * Finalizes a temporal Cross transformation by applying a
- * {@link CrossFunction} to each pair of crossed elements.<br/>
- * Each CrossFunction call returns exactly one element.
- *
- * @param function
- * The CrossFunction that is called for each pair of crossed
- * elements.
- * @return The crossed data streams
- *
- */
- public <R> SingleOutputStreamOperator<R, ?> with(CrossFunction<I1, I2, R> function) {
- TypeInformation<R> outTypeInfo = TypeExtractor.getCrossReturnTypes(function,
- op.input1.getType(), op.input2.getType());
-
- CoWindowInvokable<I1, I2, R> invokable = new CoWindowInvokable<I1, I2, R>(
- new CrossWindowFunction<I1, I2, R>(clean(function)), op.windowSize,
- op.slideInterval, op.timeStamp1, op.timeStamp2);
-
- jobGraphBuilder.setInvokable(id, invokable);
-
- return setType(outTypeInfo);
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
deleted file mode 100644
index de15515..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.Keys;
-import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TupleTypeInfo;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.util.keys.KeySelectorUtil;
-
-public class StreamJoinOperator<I1, I2> extends
- TemporalOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
-
- public StreamJoinOperator(DataStream<I1> input1, DataStream<I2> input2) {
- super(input1, input2);
- }
-
- @Override
- protected JoinWindow<I1, I2> createNextWindowOperator() {
- return new JoinWindow<I1, I2>(this);
- }
-
- public static class JoinWindow<I1, I2> {
-
- private StreamJoinOperator<I1, I2> op;
- private TypeInformation<I1> type1;
-
- private JoinWindow(StreamJoinOperator<I1, I2> operator) {
- this.op = operator;
- this.type1 = op.input1.getType();
- }
-
- /**
- * Continues a temporal Join transformation. <br/>
- * Defines the {@link Tuple} fields of the first join {@link DataStream}
- * that should be used as join keys.<br/>
- * <b>Note: Fields can only be selected as join keys on Tuple
- * DataStreams.</b><br/>
- *
- * @param fields
- * The indexes of the other Tuple fields of the first join
- * DataStreams that should be used as keys.
- * @return An incomplete Join transformation. Call
- * {@link JoinPredicate#equalTo} to continue the Join.
- */
- public JoinPredicate<I1, I2> where(int... fields) {
- return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys<I1>(fields, type1), type1));
- }
-
- /**
- * Continues a temporal join transformation. <br/>
- * Defines the fields of the first join {@link DataStream} that should
- * be used as grouping keys. Fields are the names of member fields of
- * the underlying type of the data stream.
- *
- * @param fields
- * The fields of the first join DataStream that should be
- * used as keys.
- * @return An incomplete Join transformation. Call
- * {@link JoinPredicate#equalTo} to continue the Join.
- */
- public JoinPredicate<I1, I2> where(String... fields) {
- return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys<I1>(fields, type1), type1));
- }
-
- /**
- * Continues a temporal Join transformation and defines a
- * {@link KeySelector} function for the first join {@link DataStream}
- * .</br> The KeySelector function is called for each element of the
- * first DataStream and extracts a single key value on which the
- * DataStream is joined. </br>
- *
- * @param keySelector
- * The KeySelector function which extracts the key values
- * from the DataStream on which it is joined.
- * @return An incomplete Join transformation. Call
- * {@link JoinPredicate#equalTo} to continue the Join.
- */
- public <K> JoinPredicate<I1, I2> where(KeySelector<I1, K> keySelector) {
- return new JoinPredicate<I1, I2>(op, keySelector);
- }
-
- // ----------------------------------------------------------------------------------------
-
- }
-
- /**
- * Intermediate step of a temporal Join transformation. <br/>
- * To continue the Join transformation, select the join key of the second
- * input {@link DataStream} by calling {@link JoinPredicate#equalTo}
- *
- */
- public static class JoinPredicate<I1, I2> {
-
- private StreamJoinOperator<I1, I2> op;
- private KeySelector<I1, ?> keys1;
- private KeySelector<I2, ?> keys2;
- private TypeInformation<I2> type2;
-
- private JoinPredicate(StreamJoinOperator<I1, I2> operator, KeySelector<I1, ?> keys1) {
- this.op = operator;
- this.keys1 = keys1;
- this.type2 = op.input2.getType();
- }
-
- /**
- * Creates a temporal Join transformation and defines the {@link Tuple}
- * fields of the second join {@link DataStream} that should be used as
- * join keys.<br/>
- * </p> The resulting operator wraps each pair of joining elements in a
- * Tuple2<I1,I2>(first, second). To use a different wrapping function
- * use {@link JoinedStream#with(JoinFunction)}
- *
- * @param fields
- * The indexes of the Tuple fields of the second join
- * DataStream that should be used as keys.
- * @return A streaming join operator. Call {@link JoinedStream#with} to
- * apply a custom wrapping
- */
- public JoinedStream<I1, I2> equalTo(int... fields) {
- keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields, type2),
- type2);
- return createJoinOperator();
- }
-
- /**
- * Creates a temporal Join transformation and defines the fields of the
- * second join {@link DataStream} that should be used as join keys. </p>
- * The resulting operator wraps each pair of joining elements in a
- * Tuple2<I1,I2>(first, second). To use a different wrapping function
- * use {@link JoinedStream#with(JoinFunction)}
- *
- * @param fields
- * The fields of the second join DataStream that should be
- * used as keys.
- * @return A streaming join operator. Call {@link JoinedStream#with} to
- * apply a custom wrapping
- */
- public JoinedStream<I1, I2> equalTo(String... fields) {
- this.keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields,
- type2), type2);
- return createJoinOperator();
- }
-
- /**
- * Creates a temporal Join transformation and defines a
- * {@link KeySelector} function for the second join {@link DataStream}
- * .</br> The KeySelector function is called for each element of the
- * second DataStream and extracts a single key value on which the
- * DataStream is joined. </p> The resulting operator wraps each pair of
- * joining elements in a Tuple2<I1,I2>(first, second). To use a
- * different wrapping function use
- * {@link JoinedStream#with(JoinFunction)}
- *
- *
- * @param keySelector
- * The KeySelector function which extracts the key values
- * from the second DataStream on which it is joined.
- * @return A streaming join operator. Call {@link JoinedStream#with} to
- * apply a custom wrapping
- */
- public <K> JoinedStream<I1, I2> equalTo(KeySelector<I2, K> keySelector) {
- this.keys2 = keySelector;
- return createJoinOperator();
- }
-
- private JoinedStream<I1, I2> createJoinOperator() {
-
- JoinFunction<I1, I2, Tuple2<I1, I2>> joinFunction = new DefaultJoinFunction<I1, I2>();
-
- JoinWindowFunction<I1, I2, Tuple2<I1, I2>> joinWindowFunction = getJoinWindowFunction(
- joinFunction, this);
-
- TypeInformation<Tuple2<I1, I2>> outType = new TupleTypeInfo<Tuple2<I1, I2>>(
- op.input1.getType(), op.input2.getType());
-
- return new JoinedStream<I1, I2>(this, op.input1
- .groupBy(keys1)
- .connect(op.input2.groupBy(keys2))
- .addGeneralWindowCombine(joinWindowFunction, outType, op.windowSize,
- op.slideInterval, op.timeStamp1, op.timeStamp2));
- }
- }
-
- public static class JoinedStream<I1, I2> extends
- SingleOutputStreamOperator<Tuple2<I1, I2>, JoinedStream<I1, I2>> {
- private final JoinPredicate<I1, I2> predicate;
-
- private JoinedStream(JoinPredicate<I1, I2> predicate, DataStream<Tuple2<I1, I2>> ds) {
- super(ds);
- this.predicate = predicate;
- }
-
- /**
- * Completes a stream join. </p> The resulting operator wraps each pair
- * of joining elements using the user defined {@link JoinFunction}
- *
- * @return The joined data stream.
- */
- public <OUT> SingleOutputStreamOperator<OUT, ?> with(JoinFunction<I1, I2, OUT> joinFunction) {
-
- TypeInformation<OUT> outType = TypeExtractor.getJoinReturnTypes(joinFunction,
- predicate.op.input1.getType(), predicate.op.input2.getType());
-
- CoWindowInvokable<I1, I2, OUT> invokable = new CoWindowInvokable<I1, I2, OUT>(
- getJoinWindowFunction(joinFunction, predicate), predicate.op.windowSize,
- predicate.op.slideInterval, predicate.op.timeStamp1, predicate.op.timeStamp2);
-
- jobGraphBuilder.setInvokable(id, invokable);
-
- return setType(outType);
- }
- }
-
- public static final class DefaultJoinFunction<I1, I2> implements
- JoinFunction<I1, I2, Tuple2<I1, I2>> {
-
- private static final long serialVersionUID = 1L;
- private final Tuple2<I1, I2> outTuple = new Tuple2<I1, I2>();
-
- @Override
- public Tuple2<I1, I2> join(I1 first, I2 second) throws Exception {
- outTuple.f0 = first;
- outTuple.f1 = second;
- return outTuple;
- }
- }
-
- public static <I1, I2, OUT> JoinWindowFunction<I1, I2, OUT> getJoinWindowFunction(
- JoinFunction<I1, I2, OUT> joinFunction, JoinPredicate<I1, I2> predicate) {
- return new JoinWindowFunction<I1, I2, OUT>(predicate.keys1, predicate.keys2, joinFunction);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
deleted file mode 100644
index e5385f0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/TemporalOperator.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-
-public abstract class TemporalOperator<I1, I2, OP> {
-
- public final DataStream<I1> input1;
- public final DataStream<I2> input2;
-
- public long windowSize;
- public long slideInterval;
-
- public TimestampWrapper<I1> timeStamp1;
- public TimestampWrapper<I2> timeStamp2;
-
- public TemporalOperator(DataStream<I1> input1, DataStream<I2> input2) {
- if (input1 == null || input2 == null) {
- throw new NullPointerException();
- }
- this.input1 = input1.copy();
- this.input2 = input2.copy();
- }
-
- /**
- * Continues a temporal transformation.<br/>
- * Defines the window size on which the two DataStreams will be transformed.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @return An incomplete temporal transformation.
- */
- public OP onWindow(long windowSize) {
- return onWindow(windowSize, windowSize);
- }
-
- /**
- * Continues a temporal transformation.<br/>
- * Defines the window size on which the two DataStreams will be transformed.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param slideInterval
- * The slide size of the window.
- * @return An incomplete temporal transformation.
- */
- @SuppressWarnings("unchecked")
- public OP onWindow(long windowSize, long slideInterval) {
- return onWindow(windowSize, slideInterval,
- (TimestampWrapper<I1>) SystemTimestamp.getWrapper(),
- (TimestampWrapper<I2>) SystemTimestamp.getWrapper());
- }
-
- /**
- * Continues a temporal transformation.<br/>
- * Defines the window size on which the two DataStreams will be transformed.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param slideInterval
- * The slide size of the window.
- * @param timeStamp1
- * The timestamp used to extract time from the elements of the
- * first data stream.
- * @param timeStamp2
- * The timestamp used to extract time from the elements of the
- * second data stream.
- * @return An incomplete temporal transformation.
- */
- public OP onWindow(long windowSize, long slideInterval, TimestampWrapper<I1> timeStamp1,
- TimestampWrapper<I2> timeStamp2) {
-
- this.windowSize = windowSize;
- this.slideInterval = slideInterval;
-
- this.timeStamp1 = timeStamp1;
- this.timeStamp2 = timeStamp2;
-
- return createNextWindowOperator();
- }
-
- protected abstract OP createNextWindowOperator();
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
new file mode 100644
index 0000000..8422400
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.operators.CrossOperator;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
+
+public class StreamCrossOperator<I1, I2> extends
+ TemporalOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
+
+ public StreamCrossOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ super(input1, input2);
+ }
+
+ @Override
+ protected CrossWindow<I1, I2> createNextWindowOperator() {
+
+ CrossWindowFunction<I1, I2, Tuple2<I1, I2>> crossWindowFunction = new CrossWindowFunction<I1, I2, Tuple2<I1, I2>>(
+ input1.clean(new CrossOperator.DefaultCrossFunction<I1, I2>()));
+
+ return new CrossWindow<I1, I2>(this, input1.connect(input2).addGeneralWindowCombine(
+ crossWindowFunction,
+ new TupleTypeInfo<Tuple2<I1, I2>>(input1.getType(), input2.getType()), windowSize,
+ slideInterval, timeStamp1, timeStamp2));
+ }
+
+ public static class CrossWindow<I1, I2> extends
+ SingleOutputStreamOperator<Tuple2<I1, I2>, CrossWindow<I1, I2>> implements
+ TemporalWindow<CrossWindow<I1, I2>> {
+
+ private StreamCrossOperator<I1, I2> op;
+
+ public CrossWindow(StreamCrossOperator<I1, I2> op, DataStream<Tuple2<I1, I2>> ds) {
+ super(ds);
+ this.op = op;
+ }
+
+ public CrossWindow<I1, I2> every(long length, TimeUnit timeUnit) {
+ return every(timeUnit.toMillis(length));
+ }
+
+ @SuppressWarnings("unchecked")
+ public CrossWindow<I1, I2> every(long length) {
+ ((CoWindowInvokable<I1, I2, ?>) jobGraphBuilder.getInvokable(id)).setSlideSize(length);
+ return this;
+ }
+
+ /**
+ * Finalizes a temporal Cross transformation by applying a
+ * {@link CrossFunction} to each pair of crossed elements.<br/>
+ * Each CrossFunction call returns exactly one element.
+ *
+ * @param function
+ * The CrossFunction that is called for each pair of crossed
+ * elements.
+ * @return The crossed data streams
+ *
+ */
+ public <R> SingleOutputStreamOperator<R, ?> with(CrossFunction<I1, I2, R> function) {
+ TypeInformation<R> outTypeInfo = TypeExtractor.getCrossReturnTypes(function,
+ op.input1.getType(), op.input2.getType());
+
+ CoWindowInvokable<I1, I2, R> invokable = new CoWindowInvokable<I1, I2, R>(
+ new CrossWindowFunction<I1, I2, R>(clean(function)), op.windowSize,
+ op.slideInterval, op.timeStamp1, op.timeStamp2);
+
+ jobGraphBuilder.setInvokable(id, invokable);
+
+ return setType(outTypeInfo);
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
new file mode 100644
index 0000000..626b9f1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamJoinOperator.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
+
+public class StreamJoinOperator<I1, I2> extends
+ TemporalOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
+
+ public StreamJoinOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ super(input1, input2);
+ }
+
+ @Override
+ protected JoinWindow<I1, I2> createNextWindowOperator() {
+ return new JoinWindow<I1, I2>(this);
+ }
+
+ public static class JoinWindow<I1, I2> implements TemporalWindow<JoinWindow<I1, I2>> {
+
+ private StreamJoinOperator<I1, I2> op;
+ private TypeInformation<I1> type1;
+
+ private JoinWindow(StreamJoinOperator<I1, I2> operator) {
+ this.op = operator;
+ this.type1 = op.input1.getType();
+ }
+
+ /**
+ * Continues a temporal Join transformation. <br/>
+ * Defines the {@link Tuple} fields of the first join {@link DataStream}
+ * that should be used as join keys.<br/>
+ * <b>Note: Fields can only be selected as join keys on Tuple
+ * DataStreams.</b><br/>
+ *
+ * @param fields
+ * The indexes of the other Tuple fields of the first join
+ * DataStreams that should be used as keys.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public JoinPredicate<I1, I2> where(int... fields) {
+ return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<I1>(fields, type1), type1));
+ }
+
+ /**
+ * Continues a temporal join transformation. <br/>
+ * Defines the fields of the first join {@link DataStream} that should
+ * be used as grouping keys. Fields are the names of member fields of
+ * the underlying type of the data stream.
+ *
+ * @param fields
+ * The fields of the first join DataStream that should be
+ * used as keys.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public JoinPredicate<I1, I2> where(String... fields) {
+ return new JoinPredicate<I1, I2>(op, KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys<I1>(fields, type1), type1));
+ }
+
+ /**
+ * Continues a temporal Join transformation and defines a
+ * {@link KeySelector} function for the first join {@link DataStream}
+ * .</br> The KeySelector function is called for each element of the
+ * first DataStream and extracts a single key value on which the
+ * DataStream is joined. </br>
+ *
+ * @param keySelector
+ * The KeySelector function which extracts the key values
+ * from the DataStream on which it is joined.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public <K> JoinPredicate<I1, I2> where(KeySelector<I1, K> keySelector) {
+ return new JoinPredicate<I1, I2>(op, keySelector);
+ }
+
+ @Override
+ public JoinWindow<I1, I2> every(long length, TimeUnit timeUnit) {
+ return every(timeUnit.toMillis(length));
+ }
+
+ @Override
+ public JoinWindow<I1, I2> every(long length) {
+ op.slideInterval = length;
+ return this;
+ }
+
+ // ----------------------------------------------------------------------------------------
+
+ }
+
+ /**
+ * Intermediate step of a temporal Join transformation. <br/>
+ * To continue the Join transformation, select the join key of the second
+ * input {@link DataStream} by calling {@link JoinPredicate#equalTo}
+ *
+ */
+ public static class JoinPredicate<I1, I2> {
+
+ private StreamJoinOperator<I1, I2> op;
+ private KeySelector<I1, ?> keys1;
+ private KeySelector<I2, ?> keys2;
+ private TypeInformation<I2> type2;
+
+ private JoinPredicate(StreamJoinOperator<I1, I2> operator, KeySelector<I1, ?> keys1) {
+ this.op = operator;
+ this.keys1 = keys1;
+ this.type2 = op.input2.getType();
+ }
+
+ /**
+ * Creates a temporal Join transformation and defines the {@link Tuple}
+ * fields of the second join {@link DataStream} that should be used as
+ * join keys.<br/>
+ * </p> The resulting operator wraps each pair of joining elements in a
+ * Tuple2<I1,I2>(first, second). To use a different wrapping function
+ * use {@link JoinedStream#with(JoinFunction)}
+ *
+ * @param fields
+ * The indexes of the Tuple fields of the second join
+ * DataStream that should be used as keys.
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
+ */
+ public JoinedStream<I1, I2> equalTo(int... fields) {
+ keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields, type2),
+ type2);
+ return createJoinOperator();
+ }
+
+ /**
+ * Creates a temporal Join transformation and defines the fields of the
+ * second join {@link DataStream} that should be used as join keys. </p>
+ * The resulting operator wraps each pair of joining elements in a
+ * Tuple2<I1,I2>(first, second). To use a different wrapping function
+ * use {@link JoinedStream#with(JoinFunction)}
+ *
+ * @param fields
+ * The fields of the second join DataStream that should be
+ * used as keys.
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
+ */
+ public JoinedStream<I1, I2> equalTo(String... fields) {
+ this.keys2 = KeySelectorUtil.getSelectorForKeys(new Keys.ExpressionKeys<I2>(fields,
+ type2), type2);
+ return createJoinOperator();
+ }
+
+ /**
+ * Creates a temporal Join transformation and defines a
+ * {@link KeySelector} function for the second join {@link DataStream}
+ * .</br> The KeySelector function is called for each element of the
+ * second DataStream and extracts a single key value on which the
+ * DataStream is joined. </p> The resulting operator wraps each pair of
+ * joining elements in a Tuple2<I1,I2>(first, second). To use a
+ * different wrapping function use
+ * {@link JoinedStream#with(JoinFunction)}
+ *
+ *
+ * @param keySelector
+ * The KeySelector function which extracts the key values
+ * from the second DataStream on which it is joined.
+ * @return A streaming join operator. Call {@link JoinedStream#with} to
+ * apply a custom wrapping
+ */
+ public <K> JoinedStream<I1, I2> equalTo(KeySelector<I2, K> keySelector) {
+ this.keys2 = keySelector;
+ return createJoinOperator();
+ }
+
+ private JoinedStream<I1, I2> createJoinOperator() {
+
+ JoinFunction<I1, I2, Tuple2<I1, I2>> joinFunction = new DefaultJoinFunction<I1, I2>();
+
+ JoinWindowFunction<I1, I2, Tuple2<I1, I2>> joinWindowFunction = getJoinWindowFunction(
+ joinFunction, this);
+
+ TypeInformation<Tuple2<I1, I2>> outType = new TupleTypeInfo<Tuple2<I1, I2>>(
+ op.input1.getType(), op.input2.getType());
+
+ return new JoinedStream<I1, I2>(this, op.input1
+ .groupBy(keys1)
+ .connect(op.input2.groupBy(keys2))
+ .addGeneralWindowCombine(joinWindowFunction, outType, op.windowSize,
+ op.slideInterval, op.timeStamp1, op.timeStamp2));
+ }
+ }
+
+ public static class JoinedStream<I1, I2> extends
+ SingleOutputStreamOperator<Tuple2<I1, I2>, JoinedStream<I1, I2>> {
+ private final JoinPredicate<I1, I2> predicate;
+
+ private JoinedStream(JoinPredicate<I1, I2> predicate, DataStream<Tuple2<I1, I2>> ds) {
+ super(ds);
+ this.predicate = predicate;
+ }
+
+ /**
+ * Completes a stream join. </p> The resulting operator wraps each pair
+ * of joining elements using the user defined {@link JoinFunction}
+ *
+ * @return The joined data stream.
+ */
+ public <OUT> SingleOutputStreamOperator<OUT, ?> with(JoinFunction<I1, I2, OUT> joinFunction) {
+
+ TypeInformation<OUT> outType = TypeExtractor.getJoinReturnTypes(joinFunction,
+ predicate.op.input1.getType(), predicate.op.input2.getType());
+
+ CoWindowInvokable<I1, I2, OUT> invokable = new CoWindowInvokable<I1, I2, OUT>(
+ getJoinWindowFunction(joinFunction, predicate), predicate.op.windowSize,
+ predicate.op.slideInterval, predicate.op.timeStamp1, predicate.op.timeStamp2);
+
+ jobGraphBuilder.setInvokable(id, invokable);
+
+ return setType(outType);
+ }
+ }
+
+ public static final class DefaultJoinFunction<I1, I2> implements
+ JoinFunction<I1, I2, Tuple2<I1, I2>> {
+
+ private static final long serialVersionUID = 1L;
+ private final Tuple2<I1, I2> outTuple = new Tuple2<I1, I2>();
+
+ @Override
+ public Tuple2<I1, I2> join(I1 first, I2 second) throws Exception {
+ outTuple.f0 = first;
+ outTuple.f1 = second;
+ return outTuple;
+ }
+ }
+
+ public static <I1, I2, OUT> JoinWindowFunction<I1, I2, OUT> getJoinWindowFunction(
+ JoinFunction<I1, I2, OUT> joinFunction, JoinPredicate<I1, I2> predicate) {
+ return new JoinWindowFunction<I1, I2, OUT>(predicate.keys1, predicate.keys2, joinFunction);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
new file mode 100644
index 0000000..f121dfa
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalOperator.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.windowing.helper.SystemTimestamp;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+
+public abstract class TemporalOperator<I1, I2, OP extends TemporalWindow<OP>> {
+
+ public final DataStream<I1> input1;
+ public final DataStream<I2> input2;
+
+ public long windowSize;
+ public long slideInterval;
+
+ public TimestampWrapper<I1> timeStamp1;
+ public TimestampWrapper<I2> timeStamp2;
+
+ public TemporalOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ if (input1 == null || input2 == null) {
+ throw new NullPointerException();
+ }
+ this.input1 = input1.copy();
+ this.input2 = input2.copy();
+ }
+
+ /**
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be transformed.
+ * To define sliding windows call {@link TemporalWindow#every} on the
+ * resulting operator.
+ *
+ * @param length
+ * The size of the window in milliseconds.
+ * @param timeUnit
+ * The unit if time to be used
+ * @return An incomplete temporal transformation.
+ */
+ @SuppressWarnings("unchecked")
+ public OP onWindow(long length, TimeUnit timeUnit) {
+ return onWindow(timeUnit.toMillis(length),
+ (TimestampWrapper<I1>) SystemTimestamp.getWrapper(),
+ (TimestampWrapper<I2>) SystemTimestamp.getWrapper());
+ }
+
+ /**
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be
+ * transformed.To define sliding windows call {@link TemporalWindow#every}
+ * on the resulting operator.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param timeStamp1
+ * The timestamp used to extract time from the elements of the
+ * first data stream.
+ * @param timeStamp2
+ * The timestamp used to extract time from the elements of the
+ * second data stream.
+ * @return An incomplete temporal transformation.
+ */
+ public OP onWindow(long length, Timestamp<I1> timeStamp1, Timestamp<I2> timeStamp2) {
+ return onWindow(length, timeStamp1, timeStamp2, 0);
+ }
+
+ /**
+ * Continues a temporal transformation.<br/>
+ * Defines the window size on which the two DataStreams will be
+ * transformed.To define sliding windows call {@link TemporalWindow#every}
+ * on the resulting operator.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param timeStamp1
+ * The timestamp used to extract time from the elements of the
+ * first data stream.
+ * @param timeStamp2
+ * The timestamp used to extract time from the elements of the
+ * second data stream.
+ * @param startTime
+ * The start time to measure the first window
+ * @return An incomplete temporal transformation.
+ */
+ public OP onWindow(long length, Timestamp<I1> timeStamp1, Timestamp<I2> timeStamp2,
+ long startTime) {
+ return onWindow(length, new TimestampWrapper<I1>(timeStamp1, startTime),
+ new TimestampWrapper<I2>(timeStamp2, startTime));
+ }
+
+ private OP onWindow(long length, TimestampWrapper<I1> timeStamp1,
+ TimestampWrapper<I2> timeStamp2) {
+
+ this.windowSize = length;
+ this.slideInterval = length;
+
+ this.timeStamp1 = timeStamp1;
+ this.timeStamp2 = timeStamp2;
+
+ return createNextWindowOperator();
+ }
+
+ protected abstract OP createNextWindowOperator();
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
new file mode 100644
index 0000000..8ac1492
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/TemporalWindow.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream.temporaloperator;
+
+import java.util.concurrent.TimeUnit;
+
+public interface TemporalWindow<T> {
+
+ /**
+ * Defines the slide interval for this temporal operator
+ *
+ * @param length
+ * Length of the window
+ * @param timeUnit
+ * Unit of time
+ * @return The temporal operator with slide interval specified
+ */
+ public T every(long length, TimeUnit timeUnit);
+
+ /**
+ * Defines the slide interval for this temporal operator
+ *
+ * @param length
+ * Length of the window
+ * @return The temporal operator with slide interval specified
+ */
+ public T every(long length);
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
index 59552f4..03219b7 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowInvokable.java
@@ -190,4 +190,8 @@ public class CoWindowInvokable<IN1, IN2, OUT> extends CoInvokable<IN1, IN2, OUT>
protected void callUserFunction2() throws Exception {
}
+ public void setSlideSize(long slideSize) {
+ this.slideSize = slideSize;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
index 9dc1c8c..f94eea4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -35,13 +35,13 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
*/
public class Time<DATA> implements WindowingHelper<DATA> {
- private long length;
- private TimeUnit granularity;
- private TimestampWrapper<DATA> timestampWrapper;
- private long delay;
+ protected long length;
+ protected TimeUnit granularity;
+ protected TimestampWrapper<DATA> timestampWrapper;
+ protected long delay;
/**
- * Creates an helper representing a trigger which triggers every given
+ * Creates a helper representing a trigger which triggers every given
* length or an eviction which evicts all elements older than length.
*
* @param length
@@ -62,7 +62,7 @@ public class Time<DATA> implements WindowingHelper<DATA> {
}
/**
- * Creates an helper representing a trigger which triggers every given
+ * Creates a helper representing a trigger which triggers every given
* length or an eviction which evicts all elements older than length.
*
* @param length
@@ -160,11 +160,7 @@ public class Time<DATA> implements WindowingHelper<DATA> {
return this;
}
- private long granularityInMillis() {
- if (granularity != null) {
- return this.granularity.toMillis(this.length);
- } else {
- return this.length;
- }
+ protected long granularityInMillis() {
+ return granularity == null ? length : granularity.toMillis(length);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
index 056d0a8..b52418d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -29,7 +29,7 @@ import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.sink.SinkFunction;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
import org.junit.Test;
public class WindowCrossJoinTest implements Serializable {
@@ -98,14 +98,14 @@ public class WindowCrossJoinTest implements Serializable {
inStream1
.join(inStream2)
- .onWindow(1000, 1000, new MyTimestamp<Tuple2<Integer, String>>(),
- new MyTimestamp<Tuple1<Integer>>()).where(0).equalTo(0)
+ .onWindow(1000, new MyTimestamp<Tuple2<Integer, String>>(),
+ new MyTimestamp<Tuple1<Integer>>(), 100).where(0).equalTo(0)
.addSink(new JoinResultSink());
inStream1
.cross(inStream2)
- .onWindow(1000, 1000, new MyTimestamp<Tuple2<Integer, String>>(),
- new MyTimestamp<Tuple1<Integer>>())
+ .onWindow(1000, new MyTimestamp<Tuple2<Integer, String>>(),
+ new MyTimestamp<Tuple1<Integer>>(), 100)
.with(new CrossFunction<Tuple2<Integer, String>, Tuple1<Integer>, Tuple2<Tuple2<Integer, String>, Tuple1<Integer>>>() {
private static final long serialVersionUID = 1L;
@@ -123,22 +123,13 @@ public class WindowCrossJoinTest implements Serializable {
assertEquals(crossExpectedResults, crossResults);
}
- private static class MyTimestamp<T> extends TimestampWrapper<T> {
- public MyTimestamp() {
- super(null, 0);
- }
-
+ private static class MyTimestamp<T> implements Timestamp<T> {
private static final long serialVersionUID = 1L;
@Override
public long getTimestamp(T value) {
return 101L;
}
-
- @Override
- public long getStartTime() {
- return 100L;
- }
}
private static class JoinResultSink implements
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
index 897ad48..dcfed50 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/join/WindowJoin.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.examples.join;
import java.util.Random;
+import java.util.concurrent.TimeUnit;
import org.apache.flink.api.common.functions.JoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -67,7 +68,7 @@ public class WindowJoin {
// second windows
DataStream<Tuple3<String, Integer, Integer>> joinedStream = grades
.join(salaries)
- .onWindow(1000)
+ .onWindow(1, TimeUnit.SECONDS)
.where(0)
.equalTo(0)
.with(new MyJoinFunction());
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
index 73fe1cc..b109330 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
@@ -22,6 +22,7 @@ import org.apache.flink.api.scala._
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.util.Collector
import scala.util.Random
+import java.util.concurrent.TimeUnit
object WindowJoin {
@@ -37,10 +38,10 @@ object WindowJoin {
val names = env.addSource(nameStream _).map(x => Name(x._1, x._2))
val ages = env.addSource(ageStream _).map(x => Age(x._1, x._2))
- //Join the two input streams by id on the last second and create new Person objects
- //containing both name and age
+ //Join the two input streams by id on the last second every 2 seconds and create new
+ //Person objects containing both name and age
val joined =
- names.join(ages).onWindow(1000)
+ names.join(ages).onWindow(1, TimeUnit.SECONDS).every(2, TimeUnit.SECONDS)
.where("id").equalTo("id") { (n, a) => Person(n.name, a.age) }
joined print
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
index a96c6ee..67ac8ae 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
@@ -19,7 +19,6 @@
package org.apache.flink.streaming.api.scala
import scala.reflect.ClassTag
-
import org.apache.commons.lang.Validate
import org.apache.flink.api.common.functions.CrossFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
@@ -28,11 +27,12 @@ import org.apache.flink.api.scala._
import org.apache.flink.api.scala.typeutils.CaseClassSerializer
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream}
-import org.apache.flink.streaming.api.datastream.TemporalOperator
import org.apache.flink.streaming.api.function.co.CrossWindowFunction
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
import org.apache.flink.streaming.api.scala.StreamingConversions._
+import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
+import java.util.concurrent.TimeUnit
class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
@@ -72,7 +72,7 @@ object StreamCrossOperator {
private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2],
javaStream: JavaStream[(I1, I2)]) extends
- DataStream[(I1, I2)](javaStream) {
+ DataStream[(I1, I2)](javaStream) with TemporalWindow[CrossWindow[I1, I2]] {
/**
* Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf
@@ -90,6 +90,17 @@ object StreamCrossOperator {
javaStream.setType(implicitly[TypeInformation[R]])
}
+
+ override def every(length: Long, timeUnit: TimeUnit): CrossWindow[I1, I2] = {
+ every(timeUnit.toMillis(length))
+ }
+
+ override def every(length: Long): CrossWindow[I1, I2] = {
+ val builder = javaStream.getExecutionEnvironment().getJobGraphBuilder()
+ val invokable = builder.getInvokable(javaStream.getId())
+ invokable.asInstanceOf[CoWindowInvokable[_,_,_]].setSlideSize(length)
+ this
+ }
}
private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2],
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
index 0d2a7f5..0d95c5e 100644
--- a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamJoinOperator.scala
@@ -20,25 +20,24 @@ package org.apache.flink.streaming.api.scala
import scala.Array.canBuildFrom
import scala.reflect.ClassTag
-
import org.apache.commons.lang.Validate
import org.apache.flink.api.common.functions.JoinFunction
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.common.typeutils.TypeSerializer
import org.apache.flink.api.java.functions.KeySelector
import org.apache.flink.api.java.operators.Keys
-import org.apache.flink.api.scala._
import org.apache.flink.api.scala.typeutils.CaseClassSerializer
import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
-import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream}
-import org.apache.flink.streaming.api.datastream.TemporalOperator
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
import org.apache.flink.streaming.api.function.co.JoinWindowFunction
import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
-import org.apache.flink.streaming.api.scala.StreamingConversions._
+import org.apache.flink.streaming.api.scala.StreamingConversions.javaToScalaStream
import org.apache.flink.streaming.util.keys.KeySelectorUtil
+import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
+import java.util.concurrent.TimeUnit
-class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
+class StreamJoinOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
override def createNextWindowOperator() = {
@@ -48,10 +47,11 @@ TemporalOperator[I1, I2, StreamJoinOperator.JoinWindow[I1, I2]](i1, i2) {
object StreamJoinOperator {
- class JoinWindow[I1, I2](private[flink] op: StreamJoinOperator[I1, I2]) {
+ class JoinWindow[I1, I2](private[flink]op: StreamJoinOperator[I1, I2]) extends
+ TemporalWindow[JoinWindow[I1, I2]] {
private[flink] val type1 = op.input1.getType();
-
+
/**
* Continues a temporal Join transformation by defining
* the fields in the first stream to be used as keys for the join.
@@ -60,7 +60,7 @@ object StreamJoinOperator {
*/
def where(fields: Int*) = {
new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(fields.toArray,type1),type1))
+ new Keys.ExpressionKeys(fields.toArray, type1), type1))
}
/**
@@ -69,9 +69,9 @@ object StreamJoinOperator {
* The resulting incomplete join can be completed by JoinPredicate.equalTo()
* to define the second key.
*/
- def where(firstField: String, otherFields: String*) =
+ def where(firstField: String, otherFields: String*) =
new JoinPredicate[I1, I2](op, KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(firstField +: otherFields.toArray,type1),type1))
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray, type1), type1))
/**
* Continues a temporal Join transformation by defining
@@ -89,10 +89,19 @@ object StreamJoinOperator {
new JoinPredicate[I1, I2](op, keyExtractor)
}
+ override def every(length: Long, timeUnit: TimeUnit): JoinWindow[I1, I2] = {
+ every(timeUnit.toMillis(length))
+ }
+
+ override def every(length: Long): JoinWindow[I1, I2] = {
+ op.slideInterval = length
+ this
+ }
+
}
class JoinPredicate[I1, I2](private[flink] val op: StreamJoinOperator[I1, I2],
- private[flink] val keys1: KeySelector[I1, _]) {
+ private[flink] val keys1: KeySelector[I1, _]) {
private[flink] var keys2: KeySelector[I2, _] = null
private[flink] val type2 = op.input2.getType();
@@ -104,7 +113,7 @@ object StreamJoinOperator {
*/
def equalTo(fields: Int*): JoinedStream[I1, I2] = {
finish(KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(fields.toArray,type2),type2))
+ new Keys.ExpressionKeys(fields.toArray, type2), type2))
}
/**
@@ -113,9 +122,9 @@ object StreamJoinOperator {
* (first, second)
* To define a custom wrapping, use JoinedStream.apply(...)
*/
- def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
- finish(KeySelectorUtil.getSelectorForKeys(
- new Keys.ExpressionKeys(firstField +: otherFields.toArray,type2),type2))
+ def equalTo(firstField: String, otherFields: String*): JoinedStream[I1, I2] =
+ finish(KeySelectorUtil.getSelectorForKeys(
+ new Keys.ExpressionKeys(firstField +: otherFields.toArray, type2), type2))
/**
* Creates a temporal join transformation by defining the second join key.
@@ -159,11 +168,11 @@ object StreamJoinOperator {
return op.input1.groupBy(keys1).connect(op.input2.groupBy(keys2))
.addGeneralWindowCombine(getJoinWindowFunction(this, (_, _)),
- returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
+ returnType, op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2)
}
}
- class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends
+ class JoinedStream[I1, I2](jp: JoinPredicate[I1, I2], javaStream: JavaStream[(I1, I2)]) extends
DataStream[(I1, I2)](javaStream) {
private val op = jp.op
@@ -186,7 +195,7 @@ object StreamJoinOperator {
}
private[flink] def getJoinWindowFunction[I1, I2, R](jp: JoinPredicate[I1, I2],
- joinFunction: (I1, I2) => R) = {
+ joinFunction: (I1, I2) => R) = {
Validate.notNull(joinFunction, "Join function must not be null.")
val joinFun = new JoinFunction[I1, I2, R] {
http://git-wip-us.apache.org/repos/asf/flink/blob/fb86fde2/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
new file mode 100644
index 0000000..b3343f6
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/TemporalOperator.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.streaming.api.datastream.temporaloperator.{ TemporalOperator => JTempOp }
+import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
+import org.apache.flink.streaming.api.datastream.temporaloperator.TemporalWindow
+import org.apache.flink.streaming.api.windowing.helper.Timestamp
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment._
+
+abstract class TemporalOperator[I1, I2, OP <: TemporalWindow[OP]](
+ i1: JavaStream[I1], i2: JavaStream[I2]) extends JTempOp[I1, I2, OP](i1, i2) {
+
+ def onWindow(length: Long, ts1: I1 => Long, ts2: I2 => Long, startTime: Long = 0): OP = {
+ val timeStamp1 = getTS(ts1)
+ val timeStamp2 = getTS(ts2)
+ onWindow(length, timeStamp1, timeStamp2, startTime)
+ }
+
+ def getTS[R](ts: R => Long): Timestamp[R] = {
+ new Timestamp[R] {
+ val cleanFun = clean(ts, true)
+ def getTimestamp(in: R) = cleanFun(in)
+ }
+ }
+
+}
[26/36] flink git commit: [streaming] [scala] Restructured streaming
scala project and examples
Posted by mb...@apache.org.
[streaming] [scala] Restructured streaming scala project and examples
Amended for the release-0.8 branch
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bf85c008
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bf85c008
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bf85c008
Branch: refs/heads/release-0.8
Commit: bf85c0088bb0e3198819b86aeffc5376fa3d52b7
Parents: 380934b
Author: mbalassi <mb...@apache.org>
Authored: Sun Jan 4 12:44:16 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 00:12:59 2015 +0100
----------------------------------------------------------------------
.../flink-streaming-examples/pom.xml | 137 ++++-
.../examples/windowing/TopSpeedWindowing.scala | 98 ++++
.../scala/examples/windowing/WindowJoin.scala | 71 +++
.../flink-streaming-scala/pom.xml | 202 +++++++
.../scala/ScalaStreamingAggregator.java | 111 ++++
.../api/scala/ConnectedDataStream.scala | 381 +++++++++++++
.../flink/streaming/api/scala/DataStream.scala | 558 +++++++++++++++++++
.../streaming/api/scala/SplitDataStream.scala | 50 ++
.../api/scala/StreamCrossOperator.scala | 111 ++++
.../api/scala/StreamExecutionEnvironment.scala | 275 +++++++++
.../api/scala/StreamJoinOperator.scala | 204 +++++++
.../api/scala/StreamingConversions.scala | 40 ++
.../api/scala/WindowedDataStream.scala | 209 +++++++
.../streaming/api/scala/windowing/Delta.scala | 47 ++
.../streaming/api/scala/windowing/Time.scala | 56 ++
flink-addons/flink-streaming/pom.xml | 1 +
.../streaming/windowing/TopSpeedWindowing.scala | 96 ----
.../scala/streaming/windowing/WindowJoin.scala | 71 ---
flink-scala/pom.xml | 6 -
.../streaming/ScalaStreamingAggregator.java | 111 ----
.../scala/streaming/ConnectedDataStream.scala | 380 -------------
.../flink/api/scala/streaming/DataStream.scala | 558 -------------------
.../api/scala/streaming/SplitDataStream.scala | 50 --
.../scala/streaming/StreamCrossOperator.scala | 112 ----
.../streaming/StreamExecutionEnvironment.scala | 278 ---------
.../scala/streaming/StreamJoinOperator.scala | 202 -------
.../scala/streaming/StreamingConversions.scala | 40 --
.../scala/streaming/WindowedDataStream.scala | 214 -------
.../api/scala/streaming/windowing/Delta.scala | 47 --
.../api/scala/streaming/windowing/Time.scala | 55 --
30 files changed, 2550 insertions(+), 2221 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-examples/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/pom.xml b/flink-addons/flink-streaming/flink-streaming-examples/pom.xml
index 31025cf..d483853 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/pom.xml
+++ b/flink-addons/flink-streaming/flink-streaming-examples/pom.xml
@@ -43,6 +43,12 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-scala</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-java-examples</artifactId>
<version>${project.version}</version>
</dependency>
@@ -56,7 +62,6 @@ under the License.
<build>
<plugins>
-
<!-- get default data from flink-java-examples package -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -346,6 +351,136 @@ under the License.
</execution>
</executions>
</plugin>
+
+<!-- Scala Compiler -->
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.1.4</version>
+ <executions>
+ <!-- Run scala compiler in the process-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) compile phase -->
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+
+ <!-- Run scala compiler in the process-test-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) test-compile phase -->
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <jvmArgs>
+ <jvmArg>-Xms128m</jvmArg>
+ <jvmArg>-Xmx512m</jvmArg>
+ </jvmArgs>
+ <compilerPlugins>
+ <compilerPlugin>
+ <groupId>org.scalamacros</groupId>
+ <artifactId>paradise_${scala.version}</artifactId>
+ <version>${scala.macros.version}</version>
+ </compilerPlugin>
+ </compilerPlugins>
+ </configuration>
+ </plugin>
+
+ <!-- Eclipse Integration -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.8</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ <projectnatures>
+ <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
+ <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+ </projectnatures>
+ <buildcommands>
+ <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
+ </buildcommands>
+ <classpathContainers>
+ <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
+ <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+ </classpathContainers>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>org.scala-lang:scala-compiler</exclude>
+ </excludes>
+ <sourceIncludes>
+ <sourceInclude>**/*.scala</sourceInclude>
+ <sourceInclude>**/*.java</sourceInclude>
+ </sourceIncludes>
+ </configuration>
+ </plugin>
+
+ <!-- Adding scala source directories to build path -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <!-- Add src/main/scala to eclipse build path -->
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ <!-- Add src/test/scala to eclipse build path -->
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <version>0.5.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <verbose>false</verbose>
+ <failOnViolation>true</failOnViolation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <failOnWarning>false</failOnWarning>
+ <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
+ <configLocation>${project.basedir}/../../../tools/maven/scalastyle-config.xml</configLocation>
+ <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ </plugin>
+
</plugins>
<pluginManagement>
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
new file mode 100644
index 0000000..54324e5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.windowing
+
+
+import java.util.concurrent.TimeUnit._
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment;
+import org.apache.flink.util.Collector
+import scala.math.{max, min}
+
+import scala.util.Random
+
+import org.apache.flink.streaming.api.scala.windowing.Time
+import org.apache.flink.streaming.api.scala.windowing.Delta
+
+/**
+ * An example of grouped stream windowing where different eviction and
+ * trigger policies can be used. A source fetches events from cars
+ * every 1 sec containing their id, their current speed (kmh),
+ * overall elapsed distance (m) and a timestamp. The streaming
+ * example triggers the top speed of each car every x meters elapsed
+ * for the last y seconds.
+ */
+object TopSpeedWindowing {
+
+ case class CarSpeed(carId: Int, speed: Int, distance: Double, time: Long)
+
+ def main(args: Array[String]) {
+ if (!parseParameters(args)) {
+ return
+ }
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ val cars = env.addSource(carSource _).groupBy("carId")
+ .window(Time.of(evictionSec, SECONDS))
+ .every(Delta.of[CarSpeed](triggerMeters,
+ (oldSp,newSp) => newSp.distance-oldSp.distance, CarSpeed(0,0,0,0)))
+ .reduce((x, y) => if (x.speed > y.speed) x else y)
+
+ cars print
+
+ env.execute("TopSpeedWindowing")
+
+ }
+
+ def carSource(out: Collector[CarSpeed]) = {
+
+ val speeds = new Array[Int](numOfCars)
+ val distances = new Array[Double](numOfCars)
+
+ while (true) {
+ Thread sleep 1000
+ for (i <- 0 until speeds.length) {
+ speeds(i) = if (Random.nextBoolean) min(100, speeds(i) + 5) else max(0, speeds(i) - 5)
+ distances(i) += speeds(i) / 3.6d
+ out.collect(new CarSpeed(i, speeds(i), distances(i), System.currentTimeMillis))
+ }
+ }
+ }
+
+ def parseParameters(args: Array[String]): Boolean = {
+ if (args.length > 0) {
+ if (args.length == 3) {
+ numOfCars = args(0).toInt
+ evictionSec = args(1).toInt
+ triggerMeters = args(2).toDouble
+ }
+ else {
+ System.err.println("Usage: TopSpeedWindowing <numCars> <evictSec> <triggerMeters>")
+ false
+ }
+ }
+ true
+ }
+
+ var numOfCars = 2
+ var evictionSec = 10
+ var triggerMeters = 50d
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
new file mode 100644
index 0000000..73fe1cc
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/WindowJoin.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.windowing
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
+import org.apache.flink.util.Collector
+import scala.util.Random
+
+object WindowJoin {
+
+ case class Name(id: Long, name: String)
+ case class Age(id: Long, age: Int)
+ case class Person(name: String, age: Long)
+
+ def main(args: Array[String]) {
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+
+ //Create streams for names and ages by mapping the inputs to the corresponding objects
+ val names = env.addSource(nameStream _).map(x => Name(x._1, x._2))
+ val ages = env.addSource(ageStream _).map(x => Age(x._1, x._2))
+
+ //Join the two input streams by id on the last second and create new Person objects
+ //containing both name and age
+ val joined =
+ names.join(ages).onWindow(1000)
+ .where("id").equalTo("id") { (n, a) => Person(n.name, a.age) }
+
+ joined print
+
+ env.execute("WindowJoin")
+ }
+
+ //Stream source for generating (id, name) pairs
+ def nameStream(out: Collector[(Long, String)]) = {
+ val names = Array("tom", "jerry", "alice", "bob", "john", "grace")
+
+ for (i <- 1 to 10000) {
+ if (i % 100 == 0) Thread.sleep(1000) else {
+ out.collect((i, names(Random.nextInt(names.length))))
+ }
+ }
+ }
+
+ //Stream source for generating (id, age) pairs
+ def ageStream(out: Collector[(Long, Int)]) = {
+ for (i <- 1 to 10000) {
+ if (i % 100 == 0) Thread.sleep(1000) else {
+ out.collect((i, Random.nextInt(90)))
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/pom.xml
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/pom.xml b/flink-addons/flink-streaming/flink-streaming-scala/pom.xml
new file mode 100644
index 0000000..a75d62d
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/pom.xml
@@ -0,0 +1,202 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-parent</artifactId>
+ <version>0.8-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-streaming-scala</artifactId>
+ <name>flink-streaming-scala</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-scala</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scalamacros</groupId>
+ <artifactId>quasiquotes_${scala.binary.version}</artifactId>
+ <version>${scala.macros.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm</artifactId>
+ </dependency>
+
+ <!-- guava needs to be in "provided" scope, to make sure it is not included into the jars by the shading -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- Scala Compiler -->
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <version>3.1.4</version>
+ <executions>
+ <!-- Run scala compiler in the process-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) compile phase -->
+ <execution>
+ <id>scala-compile-first</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+
+ <!-- Run scala compiler in the process-test-resources phase, so that dependencies on
+ scala classes can be resolved later in the (Java) test-compile phase -->
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <jvmArgs>
+ <jvmArg>-Xms128m</jvmArg>
+ <jvmArg>-Xmx512m</jvmArg>
+ </jvmArgs>
+ <compilerPlugins>
+ <compilerPlugin>
+ <groupId>org.scalamacros</groupId>
+ <artifactId>paradise_${scala.version}</artifactId>
+ <version>${scala.macros.version}</version>
+ </compilerPlugin>
+ </compilerPlugins>
+ </configuration>
+ </plugin>
+
+ <!-- Eclipse Integration -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-eclipse-plugin</artifactId>
+ <version>2.8</version>
+ <configuration>
+ <downloadSources>true</downloadSources>
+ <projectnatures>
+ <projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
+ <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+ </projectnatures>
+ <buildcommands>
+ <buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
+ </buildcommands>
+ <classpathContainers>
+ <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
+ <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+ </classpathContainers>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>org.scala-lang:scala-compiler</exclude>
+ </excludes>
+ <sourceIncludes>
+ <sourceInclude>**/*.scala</sourceInclude>
+ <sourceInclude>**/*.java</sourceInclude>
+ </sourceIncludes>
+ </configuration>
+ </plugin>
+
+ <!-- Adding scala source directories to build path -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <!-- Add src/main/scala to eclipse build path -->
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ <!-- Add src/test/scala to eclipse build path -->
+ <execution>
+ <id>add-test-source</id>
+ <phase>generate-test-sources</phase>
+ <goals>
+ <goal>add-test-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/test/scala</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <version>0.5.0</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <verbose>false</verbose>
+ <failOnViolation>true</failOnViolation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <failOnWarning>false</failOnWarning>
+ <sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
+ <testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
+ <configLocation>${project.basedir}/../../../tools/maven/scalastyle-config.xml</configLocation>
+ <outputFile>${project.basedir}/scalastyle-output.xml</outputFile>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java b/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
new file mode 100644
index 0000000..77d102d
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/java/org/apache/flink/api/streaming/scala/ScalaStreamingAggregator.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.streaming.scala;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.typeutils.runtime.TupleSerializerBase;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
+import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
+import org.apache.flink.streaming.api.function.aggregation.SumFunction;
+
+import scala.Product;
+
+public class ScalaStreamingAggregator<IN extends Product> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ TupleSerializerBase<IN> serializer;
+ Object[] fields;
+ int length;
+ int position;
+
+ public ScalaStreamingAggregator(TypeSerializer<IN> serializer, int pos) {
+ this.serializer = (TupleSerializerBase<IN>) serializer;
+ this.length = this.serializer.getArity();
+ this.fields = new Object[this.length];
+ this.position = pos;
+ }
+
+ public class Sum extends AggregationFunction<IN> {
+ private static final long serialVersionUID = 1L;
+ SumFunction sumFunction;
+
+ public Sum(SumFunction func) {
+ super(ScalaStreamingAggregator.this.position);
+ this.sumFunction = func;
+ }
+
+ @Override
+ public IN reduce(IN value1, IN value2) throws Exception {
+ for (int i = 0; i < length; i++) {
+ fields[i] = value2.productElement(i);
+ }
+
+ fields[position] = sumFunction.add(fields[position], value1.productElement(position));
+
+ return serializer.createInstance(fields);
+ }
+ }
+
+ public class ProductComparableAggregator extends ComparableAggregator<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ public ProductComparableAggregator(AggregationFunction.AggregationType aggregationType,
+ boolean first) {
+ super(ScalaStreamingAggregator.this.position, aggregationType, first);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public IN reduce(IN value1, IN value2) throws Exception {
+ Object v1 = value1.productElement(position);
+ Object v2 = value2.productElement(position);
+
+ int c = comparator.isExtremal((Comparable<Object>) v1, v2);
+
+ if (byAggregate) {
+ if (c == 1) {
+ return value1;
+ }
+ if (first) {
+ if (c == 0) {
+ return value1;
+ }
+ }
+
+ return value2;
+ } else {
+ for (int i = 0; i < length; i++) {
+ fields[i] = value2.productElement(i);
+ }
+
+ if (c == 1) {
+ fields[position] = v1;
+ }
+
+ return serializer.createInstance(fields);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
new file mode 100644
index 0000000..65b04b2
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/ConnectedDataStream.scala
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import java.util
+
+import scala.collection.JavaConversions.asScalaBuffer
+import scala.reflect.ClassTag
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.datastream.{ConnectedDataStream => JavaCStream}
+import org.apache.flink.streaming.api.function.co.{ CoFlatMapFunction, CoMapFunction, CoReduceFunction, CoWindowFunction }
+import org.apache.flink.streaming.api.invokable.operator.co.{ CoFlatMapInvokable, CoMapInvokable, CoReduceInvokable }
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.scala.StreamingConversions._
+import org.apache.flink.util.Collector
+
+class ConnectedDataStream[IN1, IN2](javaStream: JavaCStream[IN1, IN2]) {
+
+ /**
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * @param fun1 for each element of the first input and
+ * @param fun2 for each element of the second input. Each
+ * CoMapFunction call returns exactly one element.
+ *
+ * The CoMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def map[R: TypeInformation: ClassTag](fun1: IN1 => R, fun2: IN2 => R):
+ DataStream[R] = {
+ if (fun1 == null || fun2 == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+ val comapper = new CoMapFunction[IN1, IN2, R] {
+ def map1(in1: IN1): R = clean(fun1)(in1)
+ def map2(in2: IN2): R = clean(fun2)(in2)
+ }
+
+ new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1, IN2, R](comapper)))
+ }
+
+ /**
+ * Applies a CoMap transformation on a {@link ConnectedDataStream} and maps
+ * the output to a common type. The transformation calls a
+ * {@link CoMapFunction#map1} for each element of the first input and
+ * {@link CoMapFunction#map2} for each element of the second input. Each
+ * CoMapFunction call returns exactly one element. The user can also extend
+ * {@link RichCoMapFunction} to gain access to other features provided by
+ * the {@link RichFuntion} interface.
+ *
+ * @param coMapper
+ * The CoMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def map[R: TypeInformation: ClassTag](coMapper: CoMapFunction[IN1, IN2, R]):
+ DataStream[R] = {
+ if (coMapper == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+
+ new DataStream(javaStream.addCoFunction("map", implicitly[TypeInformation[R]],
+ new CoMapInvokable[IN1, IN2, R](coMapper)))
+ }
+
+ /**
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * {@link CoFlatMapFunction#flatMap1} for each element of the first input
+ * and {@link CoFlatMapFunction#flatMap2} for each element of the second
+ * input. Each CoFlatMapFunction call returns any number of elements
+ * including none. The user can also extend {@link RichFlatMapFunction} to
+ * gain access to other features provided by the {@link RichFuntion}
+ * interface.
+ *
+ * @param coFlatMapper
+ * The CoFlatMapFunction used to jointly transform the two input
+ * DataStreams
+ * @return The transformed { @link DataStream}
+ */
+ def flatMap[R: TypeInformation: ClassTag](coFlatMapper: CoFlatMapFunction[IN1, IN2, R]):
+ DataStream[R] = {
+ if (coFlatMapper == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ new DataStream[R](javaStream.addCoFunction("flatMap", implicitly[TypeInformation[R]],
+ new CoFlatMapInvokable[IN1, IN2, R](coFlatMapper)))
+ }
+
+ /**
+ * Applies a CoFlatMap transformation on a {@link ConnectedDataStream} and
+ * maps the output to a common type. The transformation calls a
+ * @param fun1 for each element of the first input
+ * and @param fun2 for each element of the second
+ * input. Each CoFlatMapFunction call returns any number of elements
+ * including none.
+ *
+ * @return The transformed { @link DataStream}
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun1: (IN1, Collector[R]) => Unit,
+ fun2: (IN2, Collector[R]) => Unit): DataStream[R] = {
+ if (fun1 == null || fun2 == null) {
+ throw new NullPointerException("FlatMap functions must not be null.")
+ }
+ val flatMapper = new CoFlatMapFunction[IN1, IN2, R] {
+ def flatMap1(value: IN1, out: Collector[R]): Unit = clean(fun1)(value, out)
+ def flatMap2(value: IN2, out: Collector[R]): Unit = clean(fun2)(value, out)
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPosition1 and keyPosition2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPosition1
+ * The field used to compute the hashcode of the elements in the
+ * first input stream.
+ * @param keyPosition2
+ * The field used to compute the hashcode of the elements in the
+ * second input stream.
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy(keyPosition1: Int, keyPosition2: Int): ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(keyPosition1, keyPosition2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 according to keyPositions1 and keyPositions2. Used for
+ * applying function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param keyPositions1
+ * The fields used to group the first input stream.
+ * @param keyPositions2
+ * The fields used to group the second input stream.
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy(keyPositions1: Array[Int], keyPositions2: Array[Int]):
+ ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(keyPositions1, keyPositions2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to field1 and field2. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field1
+ * The grouping expression for the first input
+ * @param field2
+ * The grouping expression for the second input
+ * @return The grouped { @link ConnectedDataStream}
+ */
+ def groupBy(field1: String, field2: String): ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(field1, field2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream using key expressions. Groups
+ * the elements of input1 and input2 according to fields1 and fields2. A
+ * field expression is either the name of a public field or a getter method
+ * with parentheses of the {@link DataStream}S underlying type. A dot can be
+ * used to drill down into objects, as in {@code "field1.getInnerField2()" }
+ * .
+ *
+ * @param fields1
+ * The grouping expressions for the first input
+ * @param fields2
+ * The grouping expressions for the second input
+ * @return The grouped { @link ConnectedDataStream}
+ */
+ def groupBy(fields1: Array[String], fields2: Array[String]):
+ ConnectedDataStream[IN1, IN2] = {
+ javaStream.groupBy(fields1, fields2)
+ }
+
+ /**
+ * GroupBy operation for connected data stream. Groups the elements of
+ * input1 and input2 using fun1 and fun2. Used for applying
+ * function on grouped data streams for example
+ * {@link ConnectedDataStream#reduce}
+ *
+ * @param fun1
+ * The function used for grouping the first input
+ * @param fun2
+ * The function used for grouping the second input
+ * @return @return The transformed { @link ConnectedDataStream}
+ */
+ def groupBy[K: TypeInformation](fun1: IN1 => _, fun2: IN2 => _):
+ ConnectedDataStream[IN1, IN2] = {
+
+ val keyExtractor1 = new KeySelector[IN1, Any] {
+ def getKey(in: IN1) = clean(fun1)(in)
+ }
+ val keyExtractor2 = new KeySelector[IN2, Any] {
+ def getKey(in: IN2) = clean(fun2)(in)
+ }
+
+ javaStream.groupBy(keyExtractor1, keyExtractor2)
+ }
+
+ /**
+ * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
+ * the outputs to a common type. If the {@link ConnectedDataStream} is
+ * batched or windowed then the reduce transformation is applied on every
+ * sliding batch/window of the data stream. If the connected data stream is
+ * grouped then the reducer is applied on every group of elements sharing
+ * the same key. This type of reduce is much faster than reduceGroup since
+ * the reduce function can be applied incrementally.
+ *
+ * @param coReducer
+ * The { @link CoReduceFunction} that will be called for every
+ * element of the inputs.
+ * @return The transformed { @link DataStream}.
+ */
+ def reduce[R: TypeInformation: ClassTag](coReducer: CoReduceFunction[IN1, IN2, R]):
+ DataStream[R] = {
+ if (coReducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+
+ new DataStream[R](javaStream.addCoFunction("coReduce", implicitly[TypeInformation[R]],
+ new CoReduceInvokable[IN1, IN2, R](coReducer)))
+ }
+
+ /**
+ * Applies a reduce transformation on a {@link ConnectedDataStream} and maps
+ * the outputs to a common type. If the {@link ConnectedDataStream} is
+ * batched or windowed then the reduce transformation is applied on every
+ * sliding batch/window of the data stream. If the connected data stream is
+ * grouped then the reducer is applied on every group of elements sharing
+ * the same key. This type of reduce is much faster than reduceGroup since
+ * the reduce function can be applied incrementally.
+ *
+ * @param reducer1
+ * @param reducer2
+ * @param mapper1
+ * @param mapper2
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def reduce[R: TypeInformation: ClassTag](reducer1: (IN1, IN1) => IN1,
+ reducer2: (IN2, IN2) => IN2,mapper1: IN1 => R, mapper2: IN2 => R): DataStream[R] = {
+ if (mapper1 == null || mapper2 == null) {
+ throw new NullPointerException("Map functions must not be null.")
+ }
+ if (reducer1 == null || reducer2 == null) {
+ throw new NullPointerException("Reduce functions must not be null.")
+ }
+
+ val reducer = new CoReduceFunction[IN1, IN2, R] {
+ def reduce1(value1: IN1, value2: IN1): IN1 = clean(reducer1)(value1, value2)
+ def map2(value: IN2): R = clean(mapper2)(value)
+ def reduce2(value1: IN2, value2: IN2): IN2 = clean(reducer2)(value1, value2)
+ def map1(value: IN1): R = clean(mapper1)(value)
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for for
+ * time aligned windows of the two data streams. System time is used as
+ * default to compute windows.
+ *
+ * @param coWindowFunction
+ * The { @link CoWindowFunction} that will be applied for the time
+ * windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def windowReduce[R: TypeInformation: ClassTag](coWindowFunction:
+ CoWindowFunction[IN1, IN2, R], windowSize: Long, slideInterval: Long) = {
+ if (coWindowFunction == null) {
+ throw new NullPointerException("CoWindow function must no be null")
+ }
+
+ javaStream.windowReduce(coWindowFunction, windowSize, slideInterval)
+ }
+
+ /**
+ * Applies a CoWindow transformation on the connected DataStreams. The
+ * transformation calls the {@link CoWindowFunction#coWindow} method for for
+ * time aligned windows of the two data streams. System time is used as
+ * default to compute windows.
+ *
+ * @param coWindower
+ * The coWindowing function to be applied for the time windows.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ *
+ * @return The transformed { @link DataStream}.
+ */
+ def windowReduce[R: TypeInformation: ClassTag](coWindower: (Seq[IN1], Seq[IN2],
+ Collector[R]) => Unit, windowSize: Long, slideInterval: Long) = {
+ if (coWindower == null) {
+ throw new NullPointerException("CoWindow function must no be null")
+ }
+
+ val coWindowFun = new CoWindowFunction[IN1, IN2, R] {
+ def coWindow(first: util.List[IN1], second: util.List[IN2],
+ out: Collector[R]): Unit = clean(coWindower)(first, second, out)
+ }
+
+ javaStream.windowReduce(coWindowFun, windowSize, slideInterval)
+ }
+
+ /**
+ * Returns the first {@link DataStream}.
+ *
+ * @return The first DataStream.
+ */
+ def getFirst(): DataStream[IN1] = {
+ javaStream.getFirst
+ }
+
+ /**
+ * Returns the second {@link DataStream}.
+ *
+ * @return The second DataStream.
+ */
+ def getSecond(): DataStream[IN2] = {
+ javaStream.getSecond
+ }
+
+ /**
+ * Gets the type of the first input
+ *
+ * @return The type of the first input
+ */
+ def getInputType1(): TypeInformation[IN1] = {
+ javaStream.getInputType1
+ }
+
+ /**
+ * Gets the type of the second input
+ *
+ * @return The type of the second input
+ */
+ def getInputType2(): TypeInformation[IN2] = {
+ javaStream.getInputType2
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
new file mode 100644
index 0000000..93444a5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -0,0 +1,558 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream,
+ SingleOutputStreamOperator, GroupedDataStream}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import scala.reflect.ClassTag
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.functions.MapFunction
+import org.apache.flink.streaming.api.invokable.operator.MapInvokable
+import org.apache.flink.util.Collector
+import org.apache.flink.api.common.functions.FlatMapFunction
+import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.streaming.api.invokable.StreamInvokable
+import org.apache.flink.streaming.api.invokable.operator.{ GroupedReduceInvokable, StreamReduceInvokable }
+import org.apache.flink.api.common.functions.ReduceFunction
+import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.common.functions.FilterFunction
+import org.apache.flink.streaming.api.function.sink.SinkFunction
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
+import org.apache.flink.streaming.api.windowing.policy.{ EvictionPolicy, TriggerPolicy }
+import org.apache.flink.streaming.api.collector.OutputSelector
+import scala.collection.JavaConversions._
+import java.util.HashMap
+import org.apache.flink.streaming.api.function.aggregation.SumFunction
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.scala.StreamingConversions._
+import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
+
+class DataStream[T](javaStream: JavaStream[T]) {
+
+ /**
+ * Gets the underlying java DataStream object.
+ */
+ def getJavaStream: JavaStream[T] = javaStream
+
+ /**
+ * Sets the degree of parallelism of this operation. This must be greater than 1.
+ */
+ def setParallelism(dop: Int): DataStream[T] = {
+ javaStream match {
+ case ds: SingleOutputStreamOperator[_, _] => ds.setParallelism(dop)
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " cannot " +
+ "have " +
+ "parallelism.")
+ }
+ this
+ }
+
+ /**
+ * Returns the degree of parallelism of this operation.
+ */
+ def getParallelism: Int = javaStream match {
+ case op: SingleOutputStreamOperator[_, _] => op.getParallelism
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " does not have" +
+ " " +
+ "parallelism.")
+ }
+
+ /**
+ * Creates a new DataStream by merging DataStream outputs of
+ * the same type with each other. The DataStreams merged using this operator
+ * will be transformed simultaneously.
+ *
+ */
+ def merge(dataStreams: DataStream[T]*): DataStream[T] =
+ javaStream.merge(dataStreams.map(_.getJavaStream): _*)
+
+ /**
+ * Creates a new ConnectedDataStream by connecting
+ * DataStream outputs of different type with each other. The
+ * DataStreams connected using this operators can be used with CoFunctions.
+ *
+ */
+ def connect[T2](dataStream: DataStream[T2]): ConnectedDataStream[T, T2] =
+ javaStream.connect(dataStream.getJavaStream)
+
+ /**
+ * Groups the elements of a DataStream by the given key positions (for tuple/array types) to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
+ def groupBy(fields: Int*): DataStream[T] = javaStream.groupBy(fields: _*)
+
+ /**
+ * Groups the elements of a DataStream by the given field expressions to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
+ def groupBy(firstField: String, otherFields: String*): DataStream[T] =
+ javaStream.groupBy(firstField +: otherFields.toArray: _*)
+
+ /**
+ * Groups the elements of a DataStream by the given K key to
+ * be used with grouped operators like grouped reduce or grouped aggregations
+ *
+ */
+ def groupBy[K: TypeInformation](fun: T => K): DataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ javaStream.groupBy(keyExtractor)
+ }
+
+ /**
+ * Sets the partitioning of the DataStream so that the output is
+ * partitioned by the selected fields. This setting only effects the how the outputs will be
+ * distributed between the parallel instances of the next processing operator.
+ *
+ */
+ def partitionBy(fields: Int*): DataStream[T] =
+ javaStream.partitionBy(fields: _*)
+
+ /**
+ * Sets the partitioning of the DataStream so that the output is
+ * partitioned by the selected fields. This setting only effects the how the outputs will be
+ * distributed between the parallel instances of the next processing operator.
+ *
+ */
+ def partitionBy(firstField: String, otherFields: String*): DataStream[T] =
+ javaStream.partitionBy(firstField +: otherFields.toArray: _*)
+
+ /**
+ * Sets the partitioning of the DataStream so that the output is
+ * partitioned by the given Key. This setting only effects the how the outputs will be
+ * distributed between the parallel instances of the next processing operator.
+ *
+ */
+ def partitionBy[K: TypeInformation](fun: T => K): DataStream[T] = {
+
+ val keyExtractor = new KeySelector[T, K] {
+ val cleanFun = clean(fun)
+ def getKey(in: T) = cleanFun(in)
+ }
+ javaStream.partitionBy(keyExtractor)
+ }
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are broadcasted to every parallel instance of the next component. This
+ * setting only effects the how the outputs will be distributed between the
+ * parallel instances of the next processing operator.
+ *
+ */
+ def broadcast: DataStream[T] = javaStream.broadcast()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are shuffled to the next component. This setting only effects the how the
+ * outputs will be distributed between the parallel instances of the next
+ * processing operator.
+ *
+ */
+ def shuffle: DataStream[T] = javaStream.shuffle()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are forwarded to the local subtask of the next component (whenever
+ * possible). This is the default partitioner setting. This setting only
+ * effects the how the outputs will be distributed between the parallel
+ * instances of the next processing operator.
+ *
+ */
+ def forward: DataStream[T] = javaStream.forward()
+
+ /**
+ * Sets the partitioning of the DataStream so that the output tuples
+ * are distributed evenly to the next component.This setting only effects
+ * the how the outputs will be distributed between the parallel instances of
+ * the next processing operator.
+ *
+ */
+ def distribute: DataStream[T] = javaStream.distribute()
+
+ /**
+ * Initiates an iterative part of the program that creates a loop by feeding
+ * back data streams. To create a streaming iteration the user needs to define
+ * a transformation that creates two DataStreams.The first one one is the output
+ * that will be fed back to the start of the iteration and the second is the output
+ * stream of the iterative part.
+ * <p>
+ * stepfunction: initialStream => (feedback, output)
+ * <p>
+ * A common pattern is to use output splitting to create feedback and output DataStream.
+ * Please refer to the .split(...) method of the DataStream
+ * <p>
+ * By default a DataStream with iteration will never terminate, but the user
+ * can use the maxWaitTime parameter to set a max waiting time for the iteration head.
+ * If no data received in the set time the stream terminates.
+ *
+ *
+ */
+ def iterate(stepFunction: DataStream[T] => (DataStream[T], DataStream[T]), maxWaitTimeMillis:
+ Long = 0): DataStream[T] = {
+ val iterativeStream = javaStream.iterate(maxWaitTimeMillis)
+
+ val (feedback, output) = stepFunction(new DataStream[T](iterativeStream))
+ iterativeStream.closeWith(feedback.getJavaStream)
+ output
+ }
+
+ /**
+ * Applies an aggregation that that gives the current maximum of the data stream at
+ * the given position.
+ *
+ */
+ def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
+
+ /**
+ * Applies an aggregation that that gives the current minimum of the data stream at
+ * the given position.
+ *
+ */
+ def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
+
+ /**
+ * Applies an aggregation that sums the data stream at the given position.
+ *
+ */
+ def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
+
+ /**
+ * Applies an aggregation that that gives the current minimum element of the data stream by
+ * the given position. When equality, the user can set to get the first or last element with
+ * the minimal value.
+ *
+ */
+ def minBy(position: Int, first: Boolean = true): DataStream[T] = aggregate(AggregationType
+ .MINBY, position, first)
+
+ /**
+ * Applies an aggregation that that gives the current maximum element of the data stream by
+ * the given position. When equality, the user can set to get the first or last element with
+ * the maximal value.
+ *
+ */
+ def maxBy(position: Int, first: Boolean = true): DataStream[T] =
+ aggregate(AggregationType.MAXBY, position, first)
+
+ private def aggregate(aggregationType: AggregationType, position: Int, first: Boolean = true):
+ DataStream[T] = {
+
+ val jStream = javaStream.asInstanceOf[JavaStream[Product]]
+ val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
+
+ val agg = new ScalaStreamingAggregator[Product](jStream.getType().createSerializer(), position)
+
+ val reducer = aggregationType match {
+ case AggregationType.SUM => new agg.Sum(SumFunction.getForClass(outType.getTypeAt(position).
+ getTypeClass()));
+ case _ => new agg.ProductComparableAggregator(aggregationType, first)
+ }
+
+ val invokable = jStream match {
+ case groupedStream: GroupedDataStream[_] => new GroupedReduceInvokable(reducer,
+ groupedStream.getKeySelector())
+ case _ => new StreamReduceInvokable(reducer)
+ }
+ new DataStream[Product](jStream.transform("aggregation", jStream.getType(),
+ invokable)).asInstanceOf[DataStream[T]]
+ }
+
+ /**
+ * Creates a new DataStream containing the current number (count) of
+ * received records.
+ *
+ */
+ def count: DataStream[Long] = new DataStream[java.lang.Long](
+ javaStream.count()).asInstanceOf[DataStream[Long]]
+
+ /**
+ * Creates a new DataStream by applying the given function to every element of this DataStream.
+ */
+ def map[R: TypeInformation: ClassTag](fun: T => R): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+ val mapper = new MapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def map(in: T): R = cleanFun(in)
+ }
+
+ javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element of this DataStream.
+ */
+ def map[R: TypeInformation: ClassTag](mapper: MapFunction[T, R]): DataStream[R] = {
+ if (mapper == null) {
+ throw new NullPointerException("Map function must not be null.")
+ }
+
+ javaStream.transform("map", implicitly[TypeInformation[R]], new MapInvokable[T, R](mapper))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](flatMapper: FlatMapFunction[T, R]): DataStream[R] = {
+ if (flatMapper == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ javaStream.transform("flatMap", implicitly[TypeInformation[R]],
+ new FlatMapInvokable[T, R](flatMapper))
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun: (T, Collector[R]) => Unit): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ val flatMapper = new FlatMapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def flatMap(in: T, out: Collector[R]) { cleanFun(in, out) }
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * Creates a new DataStream by applying the given function to every element and flattening
+ * the results.
+ */
+ def flatMap[R: TypeInformation: ClassTag](fun: T => TraversableOnce[R]): DataStream[R] = {
+ if (fun == null) {
+ throw new NullPointerException("FlatMap function must not be null.")
+ }
+ val flatMapper = new FlatMapFunction[T, R] {
+ val cleanFun = clean(fun)
+ def flatMap(in: T, out: Collector[R]) { cleanFun(in) foreach out.collect }
+ }
+ flatMap(flatMapper)
+ }
+
+ /**
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream
+ * using an associative reduce function.
+ */
+ def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
+ if (reducer == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ javaStream match {
+ case ds: GroupedDataStream[_] => javaStream.transform("reduce",
+ javaStream.getType(), new GroupedReduceInvokable[T](reducer, ds.getKeySelector()))
+ case _ => javaStream.transform("reduce", javaStream.getType(),
+ new StreamReduceInvokable[T](reducer))
+ }
+ }
+
+ /**
+ * Creates a new [[DataStream]] by reducing the elements of this DataStream
+ * using an associative reduce function.
+ */
+ def reduce(fun: (T, T) => T): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Reduce function must not be null.")
+ }
+ val reducer = new ReduceFunction[T] {
+ val cleanFun = clean(fun)
+ def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
+ }
+ reduce(reducer)
+ }
+
+ /**
+ * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
+ */
+ def filter(filter: FilterFunction[T]): DataStream[T] = {
+ if (filter == null) {
+ throw new NullPointerException("Filter function must not be null.")
+ }
+ javaStream.filter(filter)
+ }
+
+ /**
+ * Creates a new DataStream that contains only the elements satisfying the given filter predicate.
+ */
+ def filter(fun: T => Boolean): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Filter function must not be null.")
+ }
+ val filter = new FilterFunction[T] {
+ val cleanFun = clean(fun)
+ def filter(in: T) = cleanFun(in)
+ }
+ this.filter(filter)
+ }
+
+ /**
+ * Create a WindowedDataStream that can be used to apply
+ * transformation like .reduce(...) or aggregations on
+ * preset chunks(windows) of the data stream. To define the windows one or
+ * more WindowingHelper-s such as Time, Count and
+ * Delta can be used.</br></br> When applied to a grouped data
+ * stream, the windows (evictions) and slide sizes (triggers) will be
+ * computed on a per group basis. </br></br> For more advanced control over
+ * the trigger and eviction policies please use to
+ * window(List(triggers), List(evicters))
+ */
+ def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
+ javaStream.window(windowingHelper: _*)
+
+ /**
+ * Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
+ * Windowing can be used to apply transformation like .reduce(...) or aggregations on
+ * preset chunks(windows) of the data stream.</br></br>For most common
+ * use-cases please refer to window(WindowingHelper[_]*)
+ *
+ */
+ def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]):
+ WindowedDataStream[T] = javaStream.window(triggers, evicters)
+
+ /**
+ *
+ * Operator used for directing tuples to specific named outputs using an
+ * OutputSelector. Calling this method on an operator creates a new
+ * SplitDataStream.
+ */
+ def split(selector: OutputSelector[T]): SplitDataStream[T] = javaStream match {
+ case op: SingleOutputStreamOperator[_, _] => op.split(selector)
+ case _ =>
+ throw new UnsupportedOperationException("Operator " + javaStream.toString + " can not be " +
+ "split.")
+ }
+
+ /**
+ * Creates a new SplitDataStream that contains only the elements satisfying the
+ * given output selector predicate.
+ */
+ def split(fun: T => String): SplitDataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("OutputSelector must not be null.")
+ }
+ val selector = new OutputSelector[T] {
+ val cleanFun = clean(fun)
+ def select(in: T): java.lang.Iterable[String] = {
+ List(cleanFun(in))
+ }
+ }
+ split(selector)
+ }
+
+ /**
+ * Initiates a temporal Join transformation that joins the elements of two
+ * data streams on key equality over a specified time window.
+ *
+ * This method returns a StreamJoinOperator on which the
+ * .onWindow(..) should be called to define the
+ * window, and then the .where(..) and .equalTo(..) methods can be used to defin
+ * the join keys.</p> The user can also use the apply method of the returned JoinedStream
+ * to use custom join function.
+ *
+ */
+ def join[R](stream: DataStream[R]): StreamJoinOperator[T, R] =
+ new StreamJoinOperator[T, R](javaStream, stream.getJavaStream)
+
+ /**
+ * Initiates a temporal cross transformation that builds all pair
+ * combinations of elements of both DataStreams, i.e., it builds a Cartesian
+ * product.
+ *
+ * This method returns a StreamJoinOperator on which the
+ * .onWindow(..) should be called to define the
+ * window, and then the .where(..) and .equalTo(..) methods can be used to defin
+ * the join keys.</p> The user can also use the apply method of the returned JoinedStream
+ * to use custom join function.
+ *
+ */
+ def cross[R](stream: DataStream[R]): StreamCrossOperator[T, R] =
+ new StreamCrossOperator[T, R](javaStream, stream.getJavaStream)
+
+ /**
+ * Writes a DataStream to the standard output stream (stdout). For each
+ * element of the DataStream the result of .toString is
+ * written.
+ *
+ */
+ def print(): DataStream[T] = javaStream.print()
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsText(path: String, millis: Long = 0): DataStream[T] =
+ javaStream.writeAsText(path, millis)
+
+ /**
+ * Writes a DataStream to the file specified by path in text format. The
+ * writing is performed periodically, in every millis milliseconds. For
+ * every element of the DataStream the result of .toString
+ * is written.
+ *
+ */
+ def writeAsCsv(path: String, millis: Long = 0): DataStream[T] =
+ javaStream.writeAsCsv(path, millis)
+
+ /**
+ * Adds the given sink to this DataStream. Only streams with sinks added
+ * will be executed once the StreamExecutionEnvironment.execute(...)
+ * method is called.
+ *
+ */
+ def addSink(sinkFuntion: SinkFunction[T]): DataStream[T] =
+ javaStream.addSink(sinkFuntion)
+
+ /**
+ * Adds the given sink to this DataStream. Only streams with sinks added
+ * will be executed once the StreamExecutionEnvironment.execute(...)
+ * method is called.
+ *
+ */
+ def addSink(fun: T => Unit): DataStream[T] = {
+ if (fun == null) {
+ throw new NullPointerException("Sink function must not be null.")
+ }
+ val sinkFunction = new SinkFunction[T] {
+ val cleanFun = clean(fun)
+ def invoke(in: T) = cleanFun(in)
+ }
+ this.addSink(sinkFunction)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
new file mode 100644
index 0000000..a34248b
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/SplitDataStream.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
+import org.apache.flink.streaming.api.scala.StreamingConversions._
+
+/**
+ * The SplitDataStream represents an operator that has been split using an
+ * {@link OutputSelector}. Named outputs can be selected using the
+ * {@link #select} function.
+ *
+ * @param <OUT>
+ * The type of the output.
+ */
+class SplitDataStream[T](javaStream: SplitJavaStream[T]) {
+
+ /**
+ * Gets the underlying java DataStream object.
+ */
+ private[flink] def getJavaStream: SplitJavaStream[T] = javaStream
+
+ /**
+ * Sets the output names for which the next operator will receive values.
+ */
+ def select(outputNames: String*): DataStream[T] = javaStream.select(outputNames: _*)
+
+ /**
+ * Selects all output names from a split data stream.
+ */
+ def selectAll(): DataStream[T] = javaStream.selectAll()
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
new file mode 100644
index 0000000..a96c6ee
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamCrossOperator.scala
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.reflect.ClassTag
+
+import org.apache.commons.lang.Validate
+import org.apache.flink.api.common.functions.CrossFunction
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.common.typeutils.TypeSerializer
+import org.apache.flink.api.scala._
+import org.apache.flink.api.scala.typeutils.CaseClassSerializer
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
+import org.apache.flink.streaming.api.datastream.{DataStream => JavaStream}
+import org.apache.flink.streaming.api.datastream.TemporalOperator
+import org.apache.flink.streaming.api.function.co.CrossWindowFunction
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable
+import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.scala.StreamingConversions._
+
+class StreamCrossOperator[I1, I2](i1: JavaStream[I1], i2: JavaStream[I2]) extends
+ TemporalOperator[I1, I2, StreamCrossOperator.CrossWindow[I1, I2]](i1, i2) {
+
+ override def createNextWindowOperator(): StreamCrossOperator.CrossWindow[I1, I2] = {
+
+ val crossWindowFunction = StreamCrossOperator.getCrossWindowFunction(this,
+ (l: I1, r: I2) => (l, r))
+
+ val returnType = new CaseClassTypeInfo[(I1, I2)](
+
+ classOf[(I1, I2)], Seq(input1.getType, input2.getType), Array("_1", "_2")) {
+
+ override def createSerializer: TypeSerializer[(I1, I2)] = {
+ val fieldSerializers: Array[TypeSerializer[_]] = new Array[TypeSerializer[_]](getArity)
+ for (i <- 0 until getArity) {
+ fieldSerializers(i) = types(i).createSerializer
+ }
+
+ new CaseClassSerializer[(I1, I2)](classOf[(I1, I2)], fieldSerializers) {
+ override def createInstance(fields: Array[AnyRef]) = {
+ (fields(0).asInstanceOf[I1], fields(1).asInstanceOf[I2])
+ }
+ }
+ }
+ }
+
+ val javaStream = input1.connect(input2).addGeneralWindowCombine(
+ crossWindowFunction,
+ returnType, windowSize,
+ slideInterval, timeStamp1, timeStamp2);
+
+ new StreamCrossOperator.CrossWindow[I1, I2](this, javaStream)
+ }
+}
+object StreamCrossOperator {
+
+ private[flink] class CrossWindow[I1, I2](op: StreamCrossOperator[I1, I2],
+ javaStream: JavaStream[(I1, I2)]) extends
+ DataStream[(I1, I2)](javaStream) {
+
+ /**
+ * Sets a wrapper for the crossed elements. For each crossed pair, the result of the udf
+ * call will be emitted.
+ *
+ */
+ def apply[R: TypeInformation: ClassTag](fun: (I1, I2) => R): DataStream[R] = {
+
+ val invokable = new CoWindowInvokable[I1, I2, R](
+ clean(getCrossWindowFunction(op, fun)), op.windowSize, op.slideInterval, op.timeStamp1,
+ op.timeStamp2)
+
+ javaStream.getExecutionEnvironment().getJobGraphBuilder().setInvokable(javaStream.getId(),
+ invokable)
+
+ javaStream.setType(implicitly[TypeInformation[R]])
+ }
+ }
+
+ private[flink] def getCrossWindowFunction[I1, I2, R](op: StreamCrossOperator[I1, I2],
+ crossFunction: (I1, I2) => R):
+ CrossWindowFunction[I1, I2, R] = {
+ Validate.notNull(crossFunction, "Join function must not be null.")
+
+ val crossFun = new CrossFunction[I1, I2, R] {
+ val cleanFun = op.input1.clean(crossFunction)
+
+ override def cross(first: I1, second: I2): R = {
+ cleanFun(first, second)
+ }
+ }
+
+ new CrossWindowFunction[I1, I2, R](crossFun)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/bf85c008/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
new file mode 100644
index 0000000..cf05627
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/StreamExecutionEnvironment.scala
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.scala
+
+import scala.reflect.ClassTag
+
+import org.apache.commons.lang.Validate
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.scala._
+import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JavaEnv}
+import org.apache.flink.streaming.api.function.source.{ FromElementsFunction, SourceFunction }
+import org.apache.flink.streaming.api.scala.StreamingConversions.javaToScalaStream
+import org.apache.flink.util.Collector
+
+class StreamExecutionEnvironment(javaEnv: JavaEnv) {
+
+ /**
+ * Sets the degree of parallelism (DOP) for operations executed through this environment.
+ * Setting a DOP of x here will cause all operators (such as join, map, reduce) to run with
+ * x parallel instances. This value can be overridden by specific operations using
+ * [[DataStream.setParallelism]].
+ */
+ def setDegreeOfParallelism(degreeOfParallelism: Int): Unit = {
+ javaEnv.setDegreeOfParallelism(degreeOfParallelism)
+ }
+
+ /**
+ * Returns the default degree of parallelism for this execution environment. Note that this
+ * value can be overridden by individual operations using [[DataStream.setParallelism]]
+ */
+ def getDegreeOfParallelism = javaEnv.getDegreeOfParallelism
+
+ /**
+ * Sets the maximum time frequency (milliseconds) for the flushing of the
+ * output buffers. By default the output buffers flush frequently to provide
+ * low latency and to aid smooth developer experience. Setting the parameter
+ * can result in three logical modes:
+ *
+ * <ul>
+ * <li>
+ * A positive integer triggers flushing periodically by that integer</li>
+ * <li>
+ * 0 triggers flushing after every record thus minimizing latency</li>
+ * <li>
+ * -1 triggers flushing only when the output buffer is full thus maximizing
+ * throughput</li>
+ * </ul>
+ *
+ */
+ def setBufferTimeout(timeoutMillis: Long): StreamExecutionEnvironment = {
+ javaEnv.setBufferTimeout(timeoutMillis)
+ this
+ }
+
+ /**
+ * Gets the default buffer timeout set for this environment
+ */
+ def getBufferTimout: Long = javaEnv.getBufferTimeout()
+
+ /**
+ * Creates a DataStream that represents the Strings produced by reading the
+ * given file line wise. The file will be read with the system's default
+ * character set.
+ *
+ */
+ def readTextFile(filePath: String): DataStream[String] =
+ javaEnv.readTextFile(filePath)
+
+ /**
+ * Creates a DataStream that represents the Strings produced by reading the
+ * given file line wise multiple times(infinite). The file will be read with
+ * the system's default character set. This functionality can be used for
+ * testing a topology.
+ *
+ */
+ def readTextStream(StreamPath: String): DataStream[String] =
+ javaEnv.readTextStream(StreamPath)
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set.
+ *
+ */
+ def socketTextStream(hostname: String, port: Int, delimiter: Char): DataStream[String] =
+ javaEnv.socketTextStream(hostname, port, delimiter)
+
+ /**
+ * Creates a new DataStream that contains the strings received infinitely
+ * from socket. Received strings are decoded by the system's default
+ * character set, uses '\n' as delimiter.
+ *
+ */
+ def socketTextStream(hostname: String, port: Int): DataStream[String] =
+ javaEnv.socketTextStream(hostname, port)
+
+ /**
+ * Creates a new DataStream that contains a sequence of numbers.
+ *
+ */
+ def generateSequence(from: Long, to: Long): DataStream[Long] = {
+ new DataStream[java.lang.Long](javaEnv.generateSequence(from, to)).
+ asInstanceOf[DataStream[Long]]
+ }
+
+ /**
+ * Creates a DataStream that contains the given elements. The elements must all be of the
+ * same type and must be serializable.
+ *
+ * * Note that this operation will result in a non-parallel data source, i.e. a data source with
+ * a degree of parallelism of one.
+ */
+ def fromElements[T: ClassTag: TypeInformation](data: T*): DataStream[T] = {
+ val typeInfo = implicitly[TypeInformation[T]]
+ fromCollection(data)(implicitly[ClassTag[T]], typeInfo)
+ }
+
+ /**
+ * Creates a DataStream from the given non-empty [[Seq]]. The elements need to be serializable
+ * because the framework may move the elements into the cluster if needed.
+ *
+ * Note that this operation will result in a non-parallel data source, i.e. a data source with
+ * a degree of parallelism of one.
+ */
+ def fromCollection[T: ClassTag: TypeInformation](
+ data: Seq[T]): DataStream[T] = {
+ Validate.notNull(data, "Data must not be null.")
+ val typeInfo = implicitly[TypeInformation[T]]
+
+ val sourceFunction = new FromElementsFunction[T](scala.collection.JavaConversions
+ .asJavaCollection(data))
+
+ javaEnv.addSource(sourceFunction, typeInfo)
+ }
+
+ /**
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality.
+ *
+ */
+ def addSource[T: ClassTag: TypeInformation](function: SourceFunction[T]): DataStream[T] = {
+ Validate.notNull(function, "Function must not be null.")
+ val cleanFun = StreamExecutionEnvironment.clean(function)
+ val typeInfo = implicitly[TypeInformation[T]]
+ javaEnv.addSource(cleanFun, typeInfo)
+ }
+
+ /**
+ * Create a DataStream using a user defined source function for arbitrary
+ * source functionality.
+ *
+ */
+ def addSource[T: ClassTag: TypeInformation](function: Collector[T] => Unit): DataStream[T] = {
+ Validate.notNull(function, "Function must not be null.")
+ val sourceFunction = new SourceFunction[T] {
+ val cleanFun = StreamExecutionEnvironment.clean(function)
+ override def invoke(out: Collector[T]) {
+ cleanFun(out)
+ }
+ }
+ addSource(sourceFunction)
+ }
+
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with a generated
+ * default name.
+ *
+ */
+ def execute() = javaEnv.execute()
+
+ /**
+ * Triggers the program execution. The environment will execute all parts of
+ * the program that have resulted in a "sink" operation. Sink operations are
+ * for example printing results or forwarding them to a message queue.
+ * <p>
+ * The program execution will be logged and displayed with the provided name
+ *
+ */
+ def execute(jobName: String) = javaEnv.execute(jobName)
+
+}
+
+object StreamExecutionEnvironment {
+
+ private[flink] def clean[F <: AnyRef](f: F, checkSerializable: Boolean = true): F = {
+ ClosureCleaner.clean(f, checkSerializable)
+ f
+ }
+
+ /**
+ * Creates an execution environment that represents the context in which the program is
+ * currently executed. If the program is invoked standalone, this method returns a local
+ * execution environment. If the program is invoked from within the command line client
+ * to be submitted to a cluster, this method returns the execution environment of this cluster.
+ */
+ def getExecutionEnvironment: StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.getExecutionEnvironment)
+ }
+
+ /**
+ * Creates a local execution environment. The local execution environment will run the program in
+ * a multi-threaded fashion in the same JVM as the environment was created in. The default degree
+ * of parallelism of the local environment is the number of hardware contexts (CPU cores/threads).
+ */
+ def createLocalEnvironment(
+ degreeOfParallelism: Int = Runtime.getRuntime.availableProcessors()):
+ StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.createLocalEnvironment(degreeOfParallelism))
+ }
+
+ /**
+ * Creates a remote execution environment. The remote environment sends (parts of) the program to
+ * a cluster for execution. Note that all file paths used in the program must be accessible from
+ * the cluster. The execution will use the cluster's default degree of parallelism, unless the
+ * parallelism is set explicitly via [[StreamExecutionEnvironment.setDegreeOfParallelism()]].
+ *
+ * @param host The host name or address of the master (JobManager),
+ * where the program should be executed.
+ * @param port The port of the master (JobManager), where the program should be executed.
+ * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
+ * program uses
+ * user-defined functions, user-defined input formats, or any libraries,
+ * those must be
+ * provided in the JAR files.
+ */
+ def createRemoteEnvironment(host: String, port: Int, jarFiles: String*):
+ StreamExecutionEnvironment = {
+ new StreamExecutionEnvironment(JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*))
+ }
+
+ /**
+ * Creates a remote execution environment. The remote environment sends (parts of) the program
+ * to a cluster for execution. Note that all file paths used in the program must be accessible
+ * from the cluster. The execution will use the specified degree of parallelism.
+ *
+ * @param host The host name or address of the master (JobManager),
+ * where the program should be executed.
+ * @param port The port of the master (JobManager), where the program should be executed.
+ * @param degreeOfParallelism The degree of parallelism to use during the execution.
+ * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the
+ * program uses
+ * user-defined functions, user-defined input formats, or any libraries,
+ * those must be
+ * provided in the JAR files.
+ */
+ def createRemoteEnvironment(
+ host: String,
+ port: Int,
+ degreeOfParallelism: Int,
+ jarFiles: String*): StreamExecutionEnvironment = {
+ val javaEnv = JavaEnv.createRemoteEnvironment(host, port, jarFiles: _*)
+ javaEnv.setDegreeOfParallelism(degreeOfParallelism)
+ new StreamExecutionEnvironment(javaEnv)
+ }
+}
[30/36] flink git commit: [FLINK-1338] [docs] Remove incubator footer
from docs
Posted by mb...@apache.org.
[FLINK-1338] [docs] Remove incubator footer from docs
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e78c453f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e78c453f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e78c453f
Branch: refs/heads/release-0.8
Commit: e78c453f53ce55e44b98301e9562104165a7e77d
Parents: 232b0eb
Author: Stephan Ewen <se...@apache.org>
Authored: Sun Jan 4 23:08:55 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 00:22:57 2015 +0100
----------------------------------------------------------------------
docs/_includes/footer.md | 11 -----------
1 file changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e78c453f/docs/_includes/footer.md
----------------------------------------------------------------------
diff --git a/docs/_includes/footer.md b/docs/_includes/footer.md
index da4d452..2e6b13a 100644
--- a/docs/_includes/footer.md
+++ b/docs/_includes/footer.md
@@ -18,15 +18,4 @@ under the License.
-->
<hr class="divider">
-<small>Apache Flink is an effort undergoing incubation at The Apache Software
-Foundation (ASF), sponsored by the Apache Incubator PMC. Incubation is
-required of all newly accepted projects until a further review indicates that
-the infrastructure, communications, and decision making process have
-stabilized in a manner consistent with other successful ASF projects. While
-incubation status is not necessarily a reflection of the completeness or
-stability of the code, it does indicate that the project has yet to be fully
-endorsed by the ASF.</small>
-
-<a href="http://incubator.apache.org/"></a>
-
<p class="text-center"><a href="privacy-policy.html">Privacy Policy<a></p>
[27/36] flink git commit: [docs] Trivial typo in example code: o ->
out.
Posted by mb...@apache.org.
[docs] Trivial typo in example code: o -> out.
This closes #280
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/232b0ebb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/232b0ebb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/232b0ebb
Branch: refs/heads/release-0.8
Commit: 232b0ebba1a24a9637f8cf8bcf979ee7784ffa15
Parents: bf85c00
Author: Erich Schubert <kn...@users.noreply.github.com>
Authored: Fri Jan 2 19:56:57 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Tue Jan 6 00:13:08 2015 +0100
----------------------------------------------------------------------
docs/java8_programming_guide.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/232b0ebb/docs/java8_programming_guide.md
----------------------------------------------------------------------
diff --git a/docs/java8_programming_guide.md b/docs/java8_programming_guide.md
index a47ff69..da715a8 100644
--- a/docs/java8_programming_guide.md
+++ b/docs/java8_programming_guide.md
@@ -68,7 +68,7 @@ DataSet<String> input = env.fromElements(1, 2, 3);
// collector type must not be declared, it is inferred from the type of the dataset
DataSet<String> manyALetters = input.flatMap((number, out) -> {
for(int i = 0; i < number; i++) {
- o.collect("a");
+ out.collect("a");
}
});