You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2014/12/05 18:26:06 UTC
[01/34] incubator-flink git commit: [streaming] Added cosine distance
and Euclidean distance as examples for possible delta function
implementations. Test cases are included as well.
Repository: incubator-flink
Updated Branches:
refs/heads/master a677c7723 -> f84408805
[streaming] Added cosine distance and Euclidean distance as examples for possible delta function implementations. Test cases are included as well.
[streaming] Added policy helper for delta policies to provide a simpler API for such policies.
[streaming] Created ActiveEvictionPolicyWrapper. This eviction policy wrappes around a non-active policy and makes it active by forwarding notification on fake elements to the regular notification method.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/4d586e72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/4d586e72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/4d586e72
Branch: refs/heads/master
Commit: 4d586e7264194fc7fc8ca86df6e555b1e3205fac
Parents: b263624
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 14:04:36 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../windowing/deltafunction/CosineDistance.java | 87 ++++++++++++++++++++
.../deltafunction/EuclideanDistance.java | 58 +++++++++++++
.../streaming/api/windowing/helper/Delta.java | 84 +++++++++++++++++++
.../policy/ActiveEvictionPolicyWrapper.java | 63 ++++++++++++++
.../deltafunction/CosineDistanceTest.java | 70 ++++++++++++++++
.../deltafunction/EuclideanDistanceTest.java | 71 ++++++++++++++++
6 files changed, 433 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4d586e72/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
new file mode 100644
index 0000000..15aaf51
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistance.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * This delta function calculates the cosine distance between two given vectors.
+ * The cosine distance is defined as: cosineDistance=1-cosineSimilarity
+ *
+ * Cosine similarity: http://en.wikipedia.org/wiki/Cosine_similarity
+ *
+ * @param <DATA>
+ * The input data type. This delta function works with a double[],
+ * but can extract/convert to it from any other given object in case
+ * the respective extractor has been set. See
+ * {@link ExtractionAwareDeltaFunction} for more information.
+ */
+public class CosineDistance<DATA> extends ExtractionAwareDeltaFunction<DATA, double[]> {
+
+ /**
+ * auto-generated id
+ */
+ private static final long serialVersionUID = -1217813582965151599L;
+
+ public CosineDistance() {
+ super(null);
+ }
+
+ public CosineDistance(Extractor<DATA, double[]> converter) {
+ super(converter);
+ }
+
+ @Override
+ public double getNestedDelta(double[] oldDataPoint, double[] newDataPoint) {
+ if (isNullvector(oldDataPoint, newDataPoint)) {
+ return 0;
+ }
+
+ double sum1 = 0;
+ double sum2 = 0;
+ for (int i = 0; i < oldDataPoint.length; i++) {
+ sum1 += oldDataPoint[i] * oldDataPoint[i];
+ sum2 += newDataPoint[i] * newDataPoint[i];
+ }
+ sum1 = Math.sqrt(sum1);
+ sum2 = Math.sqrt(sum2);
+
+ return 1d - (dotProduct(oldDataPoint, newDataPoint) / (sum1 * sum2));
+ }
+
+ private double dotProduct(double[] a, double[] b) {
+ double result = 0;
+ for (int i = 0; i < a.length; i++) {
+ result += a[i] * b[i];
+ }
+ return result;
+ }
+
+ private boolean isNullvector(double[]... vectors) {
+ outer: for (double[] v : vectors) {
+ for (double field : v) {
+ if (field != 0) {
+ continue outer;
+ }
+ }
+ // This position is only reached in case all fields are 0.
+ return true;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4d586e72/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
new file mode 100644
index 0000000..9d055d5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistance.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * This delta function calculates the euclidean distance between two given
+ * points.
+ *
+ * Euclidean distance: http://en.wikipedia.org/wiki/Euclidean_distance
+ *
+ * @param <DATA>
+ * The input data type. This delta function works with a double[],
+ * but can extract/convert to it from any other given object in case
+ * the respective extractor has been set. See
+ * {@link ExtractionAwareDeltaFunction} for more information.
+ */
+public class EuclideanDistance<DATA> extends ExtractionAwareDeltaFunction<DATA, double[]> {
+
+ public EuclideanDistance() {
+ super(null);
+ }
+
+ public EuclideanDistance(Extractor<DATA, double[]> converter) {
+ super(converter);
+ }
+
+ /**
+ * auto-generated version id
+ */
+ private static final long serialVersionUID = 3119432599634512359L;
+
+ @Override
+ public double getNestedDelta(double[] oldDataPoint, double[] newDataPoint) {
+ double result = 0;
+ for (int i = 0; i < oldDataPoint.length; i++) {
+ result += (oldDataPoint[i] - newDataPoint[i]) * (oldDataPoint[i] - newDataPoint[i]);
+ }
+ return Math.sqrt(result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4d586e72/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
new file mode 100644
index 0000000..89c6c4a
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+import org.apache.flink.streaming.api.windowing.policy.DeltaPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * This helper represents a trigger or eviction policy based on a
+ * {@link DeltaFunction}.
+ *
+ * @param <DATA>
+ * the data type handled by the delta function represented by this
+ * helper.
+ */
+public class Delta<DATA> implements WindowingHelper<DATA> {
+
+ private DeltaFunction<DATA> deltaFunction;
+ private DATA initVal;
+ private double threshold;
+
+ /**
+ * Creates a delta helper representing a delta count or eviction policy
+ *
+ * @param deltaFunction
+ * The delta function which should be used to calculate the delta
+ * between points.
+ * @param initVal
+ * The initial value which will be used to calculate the first
+ * delta.
+ * @param threshold
+ * The threshold used by the delta function.
+ */
+ public Delta(DeltaFunction<DATA> deltaFunction, DATA initVal, double threshold) {
+ this.deltaFunction = deltaFunction;
+ this.initVal = initVal;
+ this.threshold = threshold;
+ }
+
+ @Override
+ public EvictionPolicy<DATA> toEvict() {
+ return new DeltaPolicy<DATA>(deltaFunction, initVal, threshold);
+ }
+
+ @Override
+ public TriggerPolicy<DATA> toTrigger() {
+ return new DeltaPolicy<DATA>(deltaFunction, initVal, threshold);
+ }
+
+ /**
+ * Creates a delta helper representing a delta count or eviction policy
+ *
+ * @param deltaFunction
+ * The delta function which should be used to calculate the delta
+ * between points.
+ * @param initVal
+ * The initial value which will be used to calculate the first
+ * delta.
+ * @param threshold
+ * The threshold used by the delta function.
+ * @return a delta helper representing a delta count or eviction policy
+ */
+ public static <DATA> Delta<DATA> of(DeltaFunction<DATA> deltaFunction, DATA initVal,
+ double threshold) {
+ return new Delta<DATA>(deltaFunction, initVal, threshold);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4d586e72/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
new file mode 100644
index 0000000..a110fbc
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This {@link ActiveEvictionPolicy} wraps around a non active
+ * {@link EvictionPolicy}. It forwards all calls to
+ * {@link ActiveEvictionPolicy#notifyEvictionWithFakeElement(Object, int)} to
+ * {@link EvictionPolicy#notifyEviction(Object, boolean, int)} while the
+ * triggered parameter will be set to true.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public class ActiveEvictionPolicyWrapper<DATA> implements ActiveEvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -7656558669799505882L;
+ private EvictionPolicy<DATA> nestedPolicy;
+
+ /**
+ * Creates a wrapper which activates the eviction policy which is wrapped
+ * in. This means that the nested policy will get called on fake elements as
+ * well as on real elements.
+ *
+ * @param nestedPolicy
+ * The policy which should be activated/wrapped in.
+ */
+ public ActiveEvictionPolicyWrapper(EvictionPolicy<DATA> nestedPolicy) {
+ if (nestedPolicy == null) {
+ throw new RuntimeException("The nested policy must not be null.");
+ }
+ this.nestedPolicy = nestedPolicy;
+ }
+
+ @Override
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
+ return nestedPolicy.notifyEviction(datapoint, triggered, bufferSize);
+ }
+
+ @Override
+ public int notifyEvictionWithFakeElement(DATA datapoint, int bufferSize) {
+ return nestedPolicy.notifyEviction(datapoint, true, bufferSize);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4d586e72/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
new file mode 100644
index 0000000..e12b254
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/CosineDistanceTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class CosineDistanceTest {
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCosineDistance() {
+
+ //Reference calculated using wolfram alpha
+ double[][][] testdata={
+ {{0,0,0},{0,0,0}},
+ {{0,0,0},{1,2,3}},
+ {{1,2,3},{0,0,0}},
+ {{1,2,3},{4,5,6}},
+ {{1,2,3},{-4,-5,-6}},
+ {{1,2,-3},{-4,5,-6}},
+ {{1,2,3,4},{5,6,7,8}},
+ {{1,2},{3,4}},
+ {{1},{2}},
+ };
+ double[] referenceSolutions={
+ 0,
+ 0,
+ 0,
+ 0.025368,
+ 1.974631,
+ 0.269026,
+ 0.031136,
+ 0.016130,
+ 0
+ };
+
+ for (int i = 0; i < testdata.length; i++) {
+ assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
+ + arrayToString(testdata[i][0]), referenceSolutions[i],
+ new CosineDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
+ }
+ }
+
+ private String arrayToString(double[] in){
+ if (in.length==0) return "{}";
+ String result="{";
+ for (double d:in){
+ result+=d+",";
+ }
+ return result.substring(0, result.length()-1)+"}";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4d586e72/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
new file mode 100644
index 0000000..8c62497
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/deltafunction/EuclideanDistanceTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class EuclideanDistanceTest {
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testEuclideanDistance() {
+
+ //Reference calculated using wolfram alpha
+ double[][][] testdata={
+ {{0,0,0},{0,0,0}},
+ {{0,0,0},{1,2,3}},
+ {{1,2,3},{0,0,0}},
+ {{1,2,3},{4,5,6}},
+ {{1,2,3},{-4,-5,-6}},
+ {{1,2,-3},{-4,5,-6}},
+ {{1,2,3,4},{5,6,7,8}},
+ {{1,2},{3,4}},
+ {{1},{2}},
+ };
+ double[] referenceSolutions={
+ 0,
+ 3.741657,
+ 3.741657,
+ 5.196152,
+ 12.4499,
+ 6.557439,
+ 8.0,
+ 2.828427,
+ 1
+ };
+
+ for (int i = 0; i < testdata.length; i++) {
+ assertEquals("Wrong result for inputs " + arrayToString(testdata[i][0]) + " and "
+ + arrayToString(testdata[i][0]), referenceSolutions[i],
+ new EuclideanDistance().getDelta(testdata[i][0], testdata[i][1]), 0.000001);
+ }
+
+ }
+
+ private String arrayToString(double[] in){
+ if (in.length==0) return "{}";
+ String result="{";
+ for (double d:in){
+ result+=d+",";
+ }
+ return result.substring(0, result.length()-1)+"}";
+ }
+
+}
[30/34] incubator-flink git commit: [streaming] Central
trigger/eviction API integration + invokable refactor
Posted by mb...@apache.org.
[streaming] Central trigger/eviction API integration + invokable refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/3f74e059
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/3f74e059
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/3f74e059
Branch: refs/heads/master
Commit: 3f74e0597e62e0392dae337ca26d107538544513
Parents: b082af0
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Nov 28 17:25:31 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:10 2014 +0100
----------------------------------------------------------------------
.../api/datastream/WindowedDataStream.java | 58 +++++++---
.../operator/GroupedWindowInvokable.java | 116 ++++++++++---------
.../api/invokable/operator/WindowInvokable.java | 24 ++--
.../windowing/TimeWindowingExample.java | 31 ++---
4 files changed, 136 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3f74e059/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 622c03d..15e5110 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -55,6 +55,7 @@ public class WindowedDataStream<OUT> {
protected DataStream<OUT> dataStream;
protected boolean isGrouped;
+ protected boolean allCentral;
protected KeySelector<OUT, ?> keySelector;
protected List<WindowingHelper<OUT>> triggerHelpers;
@@ -73,9 +74,13 @@ public class WindowedDataStream<OUT> {
if (dataStream instanceof GroupedDataStream) {
this.isGrouped = true;
this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+ // set all policies distributed
+ this.allCentral = false;
} else {
this.isGrouped = false;
+ // set all policies central
+ this.allCentral = true;
}
}
@@ -96,9 +101,13 @@ public class WindowedDataStream<OUT> {
if (dataStream instanceof GroupedDataStream) {
this.isGrouped = true;
this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+ // set all policies distributed
+ this.allCentral = false;
} else {
this.isGrouped = false;
+ // set all policies central
+ this.allCentral = true;
}
}
@@ -110,6 +119,7 @@ public class WindowedDataStream<OUT> {
this.evictionHelpers = windowedDataStream.evictionHelpers;
this.userTriggers = windowedDataStream.userTriggers;
this.userEvicters = windowedDataStream.userEvicters;
+ this.allCentral = windowedDataStream.allCentral;
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@@ -475,22 +485,27 @@ public class WindowedDataStream<OUT> {
private LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
- // Add Time triggers to central triggers
- for (TriggerPolicy<OUT> trigger : getTriggers()) {
- if (trigger instanceof TimeTriggerPolicy) {
- cTriggers.add(trigger);
+ if (allCentral) {
+ cTriggers.addAll(getTriggers());
+ } else {
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (trigger instanceof TimeTriggerPolicy) {
+ cTriggers.add(trigger);
+ }
}
}
return cTriggers;
}
private LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
- LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
-
- // Everything except Time triggers are distributed
- for (TriggerPolicy<OUT> trigger : getTriggers()) {
- if (!(trigger instanceof TimeTriggerPolicy)) {
- dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
+ LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = null;
+
+ if (!allCentral) {
+ dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (!(trigger instanceof TimeTriggerPolicy)) {
+ dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
+ }
}
}
@@ -498,20 +513,32 @@ public class WindowedDataStream<OUT> {
}
private LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
- LinkedList<CloneableEvictionPolicy<OUT>> evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
+ LinkedList<CloneableEvictionPolicy<OUT>> evicters = null;
- for (EvictionPolicy<OUT> evicter : getEvicters()) {
- evicters.add((CloneableEvictionPolicy<OUT>) evicter);
+ if (!allCentral) {
+ evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
+ for (EvictionPolicy<OUT> evicter : getEvicters()) {
+ evicters.add((CloneableEvictionPolicy<OUT>) evicter);
+ }
}
return evicters;
}
+ private LinkedList<EvictionPolicy<OUT>> getCentralEvicters() {
+ if (allCentral) {
+ return getEvicters();
+ } else {
+ return null;
+ }
+ }
+
private <R> StreamInvokable<OUT, R> getReduceGroupInvokable(GroupReduceFunction<OUT, R> reducer) {
StreamInvokable<OUT, R> invokable;
if (isGrouped) {
invokable = new GroupedWindowInvokable<OUT, R>(reducer, keySelector,
- getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers());
+ getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
+ getCentralEvicters());
} else {
invokable = new WindowGroupReduceInvokable<OUT, R>(reducer, getTriggers(),
@@ -524,7 +551,8 @@ public class WindowedDataStream<OUT> {
StreamInvokable<OUT, OUT> invokable;
if (isGrouped) {
invokable = new GroupedWindowInvokable<OUT, OUT>(reducer, keySelector,
- getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers());
+ getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
+ getCentralEvicters());
} else {
invokable = new WindowReduceInvokable<OUT>(reducer, getTriggers(), getEvicters());
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3f74e059/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
index 29e154b..905b45f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -83,17 +83,17 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
private KeySelector<IN, ?> keySelector;
private Configuration parameters;
- private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
- private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies;
+ private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies;
private LinkedList<ActiveEvictionPolicy<IN>> activeCentralEvictionPolicies;
private LinkedList<EvictionPolicy<IN>> centralEvictionPolicies;
- private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
- private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
- private Map<Object, WindowInvokable<IN, OUT>> windowingGroups = new HashMap<Object, WindowInvokable<IN, OUT>>();
- private LinkedList<Thread> activePolicyThreads = new LinkedList<Thread>();
- private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies;
+ private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies;
+ private Map<Object, WindowInvokable<IN, OUT>> windowingGroups;
+ private LinkedList<Thread> activePolicyThreads;
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
private LinkedList<WindowInvokable<IN, OUT>> deleteOrderForCentralEviction;
-
+
/**
* This constructor creates an instance of the grouped windowing invokable.
*
@@ -158,64 +158,70 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
super(userFunction);
- // check that not both, central and distributed eviction, is used at the
- // same time.
- if (centralEvictionPolicies != null && distributedEvictionPolicies != null
- && !centralEvictionPolicies.isEmpty() && !distributedEvictionPolicies.isEmpty()) {
- throw new UnsupportedOperationException(
- "You can only use either central or distributed eviction policies but not both at the same time.");
- }
-
- // Check that there is at least one trigger and one eviction policy
- if ((centralEvictionPolicies == null || centralEvictionPolicies.isEmpty())
- && (distributedEvictionPolicies == null || distributedEvictionPolicies.isEmpty())) {
- throw new UnsupportedOperationException(
- "You have to define at least one eviction policy");
- }
- if ((centralTriggerPolicies == null || centralTriggerPolicies.isEmpty())
- && (distributedTriggerPolicies == null || distributedTriggerPolicies.isEmpty())) {
- throw new UnsupportedOperationException(
- "You have to define at least one trigger policy");
- }
-
this.keySelector = keySelector;
// handle the triggers
- this.centralTriggerPolicies = centralTriggerPolicies;
- this.distributedTriggerPolicies = distributedTriggerPolicies;
- for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
- if (trigger instanceof ActiveTriggerPolicy) {
- this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
+ if (centralTriggerPolicies != null) {
+ this.centralTriggerPolicies = centralTriggerPolicies;
+ this.activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+
+ for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
+ if (trigger instanceof ActiveTriggerPolicy) {
+ this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
+ }
}
+ } else {
+ this.centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
}
- // handle the evictions
- if (distributedEvictionPolicies != null && !distributedEvictionPolicies.isEmpty()) {
+ if (distributedTriggerPolicies != null) {
+ this.distributedTriggerPolicies = distributedTriggerPolicies;
+ } else {
+ this.distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
+ }
+
+ if (distributedEvictionPolicies != null) {
this.distributedEvictionPolicies = distributedEvictionPolicies;
- } else { // (centralEvictionPolicies!=null&&!centralEvictionPolicies.isEmpty())
+ } else {
+ this.distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
+ }
+
+ this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+
+ if (centralEvictionPolicies != null) {
this.centralEvictionPolicies = centralEvictionPolicies;
- this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+
for (EvictionPolicy<IN> eviction : centralEvictionPolicies) {
if (eviction instanceof ActiveEvictionPolicy) {
this.activeCentralEvictionPolicies.add((ActiveEvictionPolicy<IN>) eviction);
}
}
- this.deleteOrderForCentralEviction = new LinkedList<WindowInvokable<IN, OUT>>();
+ } else {
+ this.centralEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
+ }
+
+ this.windowingGroups = new HashMap<Object, WindowInvokable<IN, OUT>>();
+ this.activePolicyThreads = new LinkedList<Thread>();
+ this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ this.deleteOrderForCentralEviction = new LinkedList<WindowInvokable<IN, OUT>>();
+
+ // check that not both, central and distributed eviction, is used at the
+ // same time.
+ if (!this.centralEvictionPolicies.isEmpty() && !this.distributedEvictionPolicies.isEmpty()) {
+ throw new UnsupportedOperationException(
+ "You can only use either central or distributed eviction policies but not both at the same time.");
+ }
+
+ // Check that there is at least one trigger and one eviction policy
+ if (this.centralEvictionPolicies.isEmpty() && this.distributedEvictionPolicies.isEmpty()) {
+ throw new UnsupportedOperationException(
+ "You have to define at least one eviction policy");
+ }
+ if (this.centralTriggerPolicies.isEmpty() && this.distributedTriggerPolicies.isEmpty()) {
+ throw new UnsupportedOperationException(
+ "You have to define at least one trigger policy");
}
- }
-
- /**
- * Same as
- * {@link GroupedWindowInvokable#GroupedWindowInvokable(Function, KeySelector, LinkedList, LinkedList, LinkedList, LinkedList)}
- * but using always distributed eviction only.
- */
- public GroupedWindowInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
- LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
- LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
- LinkedList<TriggerPolicy<IN>> centralTriggerPolicies) {
- this(userFunction, keySelector, distributedTriggerPolicies, distributedEvictionPolicies,
- centralTriggerPolicies, null);
}
@Override
@@ -239,7 +245,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
for (Object in : result) {
// If central eviction is used, handle it here
- if (activeCentralEvictionPolicies!=null) {
+ if (!activeCentralEvictionPolicies.isEmpty()) {
evictElements(centralActiveEviction(in));
}
@@ -263,7 +269,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
groupInvokable.processRealElement(reuse.getObject());
// If central eviction is used, handle it here
- if (centralEvictionPolicies!=null) {
+ if (!centralEvictionPolicies.isEmpty()) {
evictElements(centralEviction(reuse.getObject(), false));
deleteOrderForCentralEviction.add(groupInvokable);
}
@@ -283,7 +289,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
}
// If central eviction is used, handle it here
- if (centralEvictionPolicies!=null) {
+ if (!centralEvictionPolicies.isEmpty()) {
evictElements(centralEviction(reuse.getObject(), true));
deleteOrderForCentralEviction.add(groupInvokable);
}
@@ -473,7 +479,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
public void sendFakeElement(Object datapoint) {
// If central eviction is used, handle it here
- if (centralEvictionPolicies!=null) {
+ if (!centralEvictionPolicies.isEmpty()) {
evictElements(centralActiveEviction(datapoint));
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3f74e059/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
index 740fa4f..bbc1277 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -45,9 +45,9 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
private LinkedList<EvictionPolicy<IN>> evictionPolicies;
private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
- private LinkedList<Thread> activePolicyTreads = new LinkedList<Thread>();
- protected LinkedList<IN> buffer = new LinkedList<IN>();
- private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ private LinkedList<Thread> activePolicyTreads;
+ protected LinkedList<IN> buffer;
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
/**
* This constructor created a windowing invokable using trigger and eviction
@@ -82,6 +82,10 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
}
}
+
+ this.activePolicyTreads = new LinkedList<Thread>();
+ this.buffer = new LinkedList<IN>();
+ this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
}
@Override
@@ -156,12 +160,12 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
* of this group.
*
* Remark: This is NOT the same as
- * {@link WindowInvokable#processFakeElement(Object, TriggerPolicy)}!
- * Here the eviction using active policies takes place after the call to the
- * UDF. Usually it is done before when fake elements get submitted. This
- * special behaviour is needed to allow the
- * {@link GroupedWindowInvokable} to send central triggers to all groups,
- * even if the current element does not belong to the group.
+ * {@link WindowInvokable#processFakeElement(Object, TriggerPolicy)}! Here
+ * the eviction using active policies takes place after the call to the UDF.
+ * Usually it is done before when fake elements get submitted. This special
+ * behaviour is needed to allow the {@link GroupedWindowInvokable} to send
+ * central triggers to all groups, even if the current element does not
+ * belong to the group.
*
* @param input
* a fake input element
@@ -339,7 +343,7 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
buffer.add(input);
}
-
+
/**
* This method removes the first element from the element buffer. It is used
* to provide central evictions in {@link GroupedWindowInvokable}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3f74e059/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
index fba73be..128d3bf 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
@@ -36,7 +36,7 @@ import org.apache.flink.util.Collector;
*/
public class TimeWindowingExample {
- private static final int PARALLELISM = 2;
+ private static final int PARALLELISM = 1;
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment
@@ -45,21 +45,26 @@ public class TimeWindowingExample {
// Prevent output from being blocked
env.setBufferTimeout(100);
- DataStream<Integer> stream = env.addSource(new CountingSourceWithSleep())
- .groupBy(new KeySelector<Integer, Integer>() {
+ KeySelector<Integer, Integer> myKey = new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1L;
- @Override
- public Integer getKey(Integer value) throws Exception {
- if (value < 3) {
- return 0;
- } else {
- return 1;
- }
- }
+ @Override
+ public Integer getKey(Integer value) throws Exception {
+ if (value < 2) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
- }).window(Count.of(100)).every(Time.of(1000, TimeUnit.MILLISECONDS)).sum();
+ };
+
+ DataStream<Integer> stream = env.addSource(new CountingSourceWithSleep())
+ .window(Count.of(100))
+ .every(Time.of(1000, TimeUnit.MILLISECONDS))
+ .groupBy(myKey)
+ .sum();
stream.print();
[23/34] incubator-flink git commit: [streaming] Windowing helper API
cleanup
Posted by mb...@apache.org.
[streaming] Windowing helper API cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/d6dc4349
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/d6dc4349
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/d6dc4349
Branch: refs/heads/master
Commit: d6dc4349c3244f460bbc1227125bf80a3f4be4c7
Parents: 996f02c
Author: Gyula Fora <gy...@apache.org>
Authored: Tue Nov 25 18:37:25 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../streaming/api/windowing/helper/Count.java | 20 +++++-
.../streaming/api/windowing/helper/Delta.java | 2 +-
.../streaming/api/windowing/helper/Time.java | 70 +++++++++++---------
.../apache/flink/streaming/api/PrintTest.java | 44 +-----------
4 files changed, 58 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d6dc4349/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
index 6e045f6..73f7daf 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
@@ -56,11 +56,29 @@ public class Count implements WindowingHelper {
return new CountTriggerPolicy(count, startValue);
}
+ /**
+ * Sets the number of elements deleted at each eviction (i.e when the number
+ * elements exceeds the window size). By default the elements get deleted
+ * one by one (deleteOnEvition = 1)
+ *
+ * @param deleteOnEviction
+ * The number of elements deleted at each evition
+ * @return Helper representing the count based policy
+ *
+ */
public Count withDelete(int deleteOnEviction) {
this.deleteOnEviction = deleteOnEviction;
return this;
}
+ /**
+ * Sets the initial value of the counter. 0 by default
+ *
+ * @param startValue
+ * Starting value of the window counter
+ * @return Helper representing the count based policy
+ *
+ */
public Count startingAt(int startValue) {
this.startValue = startValue;
return this;
@@ -72,7 +90,7 @@ public class Count implements WindowingHelper {
*
* @param count
* the number of elements to count before trigger/evict
- * @return An helper representing the policy
+ * @return Helper representing the count based policy
*/
public static Count of(int count) {
return new Count(count);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d6dc4349/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
index 89c6c4a..ce75fc5 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
@@ -75,7 +75,7 @@ public class Delta<DATA> implements WindowingHelper<DATA> {
* delta.
* @param threshold
* The threshold used by the delta function.
- * @return a delta helper representing a delta count or eviction policy
+ * @return Helper representing a delta trigger or eviction policy
*/
public static <DATA> Delta<DATA> of(DeltaFunction<DATA> deltaFunction, DATA initVal,
double threshold) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d6dc4349/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
index 86ad952..908d0cf 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -21,10 +21,9 @@ import java.util.concurrent.TimeUnit;
import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
/**
@@ -38,27 +37,30 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
*/
public class Time<DATA> implements WindowingHelper<DATA> {
- private int timeVal;
+ private long length;
private TimeUnit granularity;
private TimeStamp<DATA> timeStamp;
private long delay;
/**
* Creates an helper representing a trigger which triggers every given
- * timeVal or an eviction which evicts all elements older than timeVal.
+ * length or an eviction which evicts all elements older than length.
*
- * @param timeVal
+ * @param length
* The number of time units
* @param timeUnit
* The unit of time such as minute oder millisecond. Note that
* the smallest possible granularity is milliseconds. Any smaller
* time unit might cause an error at runtime due to conversion
* problems.
+ * @param timeStamp
+ * The user defined timestamp that will be used to extract time
+ * information from the incoming elements
*/
- private Time(int timeVal, TimeUnit timeUnit) {
- this.timeVal = timeVal;
+ private Time(long length, TimeUnit timeUnit, TimeStamp<DATA> timeStamp) {
+ this.length = length;
this.granularity = timeUnit;
- this.timeStamp = new DefaultTimeStamp<DATA>();
+ this.timeStamp = timeStamp;
this.delay = 0;
}
@@ -74,47 +76,49 @@ public class Time<DATA> implements WindowingHelper<DATA> {
/**
* Creates an helper representing a trigger which triggers every given
- * timeVal or an eviction which evicts all elements older than timeVal.
+ * length or an eviction which evicts all elements older than length.
*
- * @param timeVal
+ * @param length
* The number of time units
- * @param granularity
+ * @param timeUnit
* The unit of time such as minute oder millisecond. Note that
* the smallest possible granularity is milliseconds. Any smaller
* time unit might cause an error at runtime due to conversion
* problems.
- * @return an helper representing a trigger which triggers every given
- * timeVal or an eviction which evicts all elements older than
- * timeVal.
+ * @return Helper representing the time based trigger and eviction policy
*/
- public static <DATA> Time<DATA> of(int timeVal, TimeUnit granularity) {
- return new Time<DATA>(timeVal, granularity);
+ public static <DATA> Time<DATA> of(long length, TimeUnit timeUnit) {
+ return new Time<DATA>(length, timeUnit, new DefaultTimeStamp<DATA>());
}
- @SuppressWarnings("unchecked")
- public <R> Time<R> withTimeStamp(TimeStamp<R> timeStamp) {
- this.timeStamp = (TimeStamp<DATA>) timeStamp;
- return (Time<R>) this;
+ /**
+ * Creates an helper representing a trigger which triggers every given
+ * length or an eviction which evicts all elements older than length.
+ *
+ * @param length
+ * The number of time units
+ * @param timeStamp
+ * The user defined timestamp that will be used to extract time
+ * information from the incoming elements
+ * @return Helper representing the time based trigger and eviction policy
+ */
+ public static <DATA> Time<DATA> of(long length, TimeStamp<DATA> timeStamp) {
+ return new Time<DATA>(length, TimeUnit.MILLISECONDS, timeStamp);
}
+ /**
+ * Sets the delay for the first processed window.
+ *
+ * @param delay
+ * The number of time units before the first processed window.
+ * @return Helper representing the time based trigger and eviction policy
+ */
public Time<DATA> withDelay(long delay) {
this.delay = delay;
return this;
}
private long granularityInMillis() {
- return this.granularity.toMillis(this.timeVal);
- }
-
- public static class NullExtractor<T> implements Extractor<Long, T> {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public T extract(Long in) {
- return null;
- }
-
+ return this.granularity.toMillis(this.length);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d6dc4349/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
index f7ba82e..dd8652e 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
@@ -18,21 +18,11 @@
package org.apache.flink.streaming.api;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.apache.flink.streaming.api.windowing.helper.Time;
-import org.apache.flink.util.Collector;
import org.junit.Test;
public class PrintTest implements Serializable {
@@ -60,39 +50,7 @@ public class PrintTest implements Serializable {
@Test
public void test() throws Exception {
LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
- List<Tuple2<String, Integer>> input = new ArrayList<Tuple2<String, Integer>>();
-
- env.fromElements(1, 2, 3, 4, 5, 6, 7, 8, 9)
- .window(Time.of(2, TimeUnit.MILLISECONDS).withTimeStamp(new TimeStamp<Integer>() {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Integer value) {
-
- return value;
- }
-
- @Override
- public long getStartTime() {
- return 1;
- }
- })).every(Count.of(2)).reduceGroup(new GroupReduceFunction<Integer, String>() {
-
- @Override
- public void reduce(Iterable<Integer> values, Collector<String> out)
- throws Exception {
- String o = "|";
- for (Integer v : values) {
- o = o + v + "|";
- }
- out.collect(o);
- }
- }).print();
+ env.generateSequence(1, 10).map(new IdentityMap()).filter(new FilterAll()).print();
env.executeTest(MEMORYSIZE);
}
[29/34] incubator-flink git commit: [streaming] Renamed test cases
for window and grouped window invokable to be consistent with the invokable
names
Posted by mb...@apache.org.
[streaming] Renamed test cases for window and grouped window invokable to be consistent with the invokable names
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/240e5f13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/240e5f13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/240e5f13
Branch: refs/heads/master
Commit: 240e5f139f53f98ee15458c113c8688ff9b71099
Parents: f0cc5d6
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Thu Nov 27 13:08:46 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:10 2014 +0100
----------------------------------------------------------------------
.../windowing/extractor/FieldsFromTuple.java | 2 +-
.../operator/GroupedWindowInvokableTest.java | 417 +++++++++++++++++++
.../operator/GroupedWindowingInvokableTest.java | 417 -------------------
.../invokable/operator/WindowInvokableTest.java | 263 ++++++++++++
.../operator/WindowingInvokableTest.java | 263 ------------
5 files changed, 681 insertions(+), 681 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/240e5f13/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
index 58bb4e0..afd0421 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
@@ -46,7 +46,7 @@ public class FieldsFromTuple implements Extractor<Tuple, double[]> {
public double[] extract(Tuple in) {
double[] out = new double[indexes.length];
for (int i = 0; i < indexes.length; i++) {
- out[i] = in.getField(indexes[i]);
+ out[i] = (Double) in.getField(indexes[i]);
}
return out;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/240e5f13/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
new file mode 100644
index 0000000..9d10166
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.MockInvokable;
+import org.apache.flink.streaming.util.keys.TupleKeySelector;
+import org.junit.Test;
+
+public class GroupedWindowInvokableTest {
+
+ /**
+ * Test for not active distributed triggers with single field
+ */
+ @Test
+ public void testGroupedWindowInvokableDistributedTriggerSimple() {
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(1);
+ inputs.add(5);
+ inputs.add(5);
+ inputs.add(5);
+ inputs.add(1);
+ inputs.add(1);
+ inputs.add(5);
+ inputs.add(1);
+ inputs.add(5);
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(15);
+ expected.add(3);
+ expected.add(3);
+ expected.add(15);
+
+ LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
+
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
+ new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ }, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, triggers, evictions, centralTriggers);
+
+ List<Integer> result = MockInvokable.createAndExecute(invokable, inputs);
+
+ List<Integer> actual = new LinkedList<Integer>();
+ for (Integer current : result) {
+ actual.add(current);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(actual));
+ assertEquals(expected.size(), actual.size());
+ }
+
+ /**
+ * Test for non active distributed triggers with separated key field
+ */
+ @Test
+ public void testGroupedWindowInvokableDistributedTriggerComplex() {
+ List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
+ inputs2.add(new Tuple2<Integer, String>(1, "a"));
+ inputs2.add(new Tuple2<Integer, String>(0, "b"));
+ inputs2.add(new Tuple2<Integer, String>(2, "a"));
+ inputs2.add(new Tuple2<Integer, String>(-1, "a"));
+ inputs2.add(new Tuple2<Integer, String>(-2, "a"));
+ inputs2.add(new Tuple2<Integer, String>(10, "a"));
+ inputs2.add(new Tuple2<Integer, String>(2, "b"));
+ inputs2.add(new Tuple2<Integer, String>(1, "a"));
+
+ List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
+ expected2.add(new Tuple2<Integer, String>(-1, "a"));
+ expected2.add(new Tuple2<Integer, String>(-2, "a"));
+ expected2.add(new Tuple2<Integer, String>(0, "b"));
+
+ LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
+
+ LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
+
+ LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+
+ GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ new ReduceFunction<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+ Tuple2<Integer, String> value2) throws Exception {
+ if (value1.f0 <= value2.f0) {
+ return value1;
+ } else {
+ return value2;
+ }
+ }
+ }, new TupleKeySelector<Tuple2<Integer, String>>(1), triggers, evictions,
+ centralTriggers);
+
+ List<Tuple2<Integer, String>> result = MockInvokable.createAndExecute(invokable2, inputs2);
+
+ List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
+ for (Tuple2<Integer, String> current : result) {
+ actual2.add(current);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
+ new HashSet<Tuple2<Integer, String>>(actual2));
+ assertEquals(expected2.size(), actual2.size());
+ }
+
+ /**
+ * Test for active centralized trigger
+ */
+ @Test
+ public void testGroupedWindowInvokableCentralActiveTrigger() {
+
+ List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
+ inputs.add(new Tuple2<Integer, String>(1, "a"));
+ inputs.add(new Tuple2<Integer, String>(1, "b"));
+ inputs.add(new Tuple2<Integer, String>(1, "c"));
+ inputs.add(new Tuple2<Integer, String>(2, "a"));
+ inputs.add(new Tuple2<Integer, String>(2, "b"));
+ inputs.add(new Tuple2<Integer, String>(2, "c"));
+ inputs.add(new Tuple2<Integer, String>(2, "b"));
+ inputs.add(new Tuple2<Integer, String>(2, "a"));
+ inputs.add(new Tuple2<Integer, String>(2, "c"));
+ inputs.add(new Tuple2<Integer, String>(3, "c"));
+ inputs.add(new Tuple2<Integer, String>(3, "a"));
+ inputs.add(new Tuple2<Integer, String>(3, "b"));
+ inputs.add(new Tuple2<Integer, String>(4, "a"));
+ inputs.add(new Tuple2<Integer, String>(4, "b"));
+ inputs.add(new Tuple2<Integer, String>(4, "c"));
+ inputs.add(new Tuple2<Integer, String>(5, "c"));
+ inputs.add(new Tuple2<Integer, String>(5, "a"));
+ inputs.add(new Tuple2<Integer, String>(5, "b"));
+ inputs.add(new Tuple2<Integer, String>(10, "b"));
+ inputs.add(new Tuple2<Integer, String>(10, "a"));
+ inputs.add(new Tuple2<Integer, String>(10, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "a"));
+ inputs.add(new Tuple2<Integer, String>(11, "a"));
+ inputs.add(new Tuple2<Integer, String>(11, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "b"));
+ inputs.add(new Tuple2<Integer, String>(11, "b"));
+
+ // Expected result:
+ // For each group (a,b and c):
+ // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+ // 12-12-5-10-32
+
+ List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
+ expected.add(new Tuple2<Integer, String>(12, "a"));
+ expected.add(new Tuple2<Integer, String>(12, "b"));
+ expected.add(new Tuple2<Integer, String>(12, "c"));
+ expected.add(new Tuple2<Integer, String>(12, "a"));
+ expected.add(new Tuple2<Integer, String>(12, "b"));
+ expected.add(new Tuple2<Integer, String>(12, "c"));
+ expected.add(new Tuple2<Integer, String>(5, "a"));
+ expected.add(new Tuple2<Integer, String>(5, "b"));
+ expected.add(new Tuple2<Integer, String>(5, "c"));
+ expected.add(new Tuple2<Integer, String>(10, "a"));
+ expected.add(new Tuple2<Integer, String>(10, "b"));
+ expected.add(new Tuple2<Integer, String>(10, "c"));
+ expected.add(new Tuple2<Integer, String>(32, "a"));
+ expected.add(new Tuple2<Integer, String>(32, "b"));
+ expected.add(new Tuple2<Integer, String>(32, "c"));
+
+ TimeStamp<Tuple2<Integer, String>> myTimeStamp = new TimeStamp<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Tuple2<Integer, String> value) {
+ return value.f0;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 1;
+ }
+ };
+
+ ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+ Tuple2<Integer, String> value2) throws Exception {
+ return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
+ }
+ };
+
+ LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+ // Trigger every 2 time units but delay the first trigger by 2 (First
+ // trigger after 4, then every 2)
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L));
+
+ LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+ // Always delete all elements older then 4
+ evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStamp));
+
+ LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+
+ GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
+ distributedTriggers, evictions, triggers);
+
+ ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
+ for (Tuple2<Integer, String> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+ /**
+ * Test for multiple centralized trigger
+ */
+ @Test
+ public void testGroupedWindowInvokableMultipleCentralTrigger() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(8));
+ triggers.add(new CountTriggerPolicy<Integer>(5));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // The active wrapper causes eviction even on (fake) elements which
+ // triggered, but does not belong to the group.
+ evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+ new TumblingEvictionPolicy<Integer>()));
+
+ LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(1);
+ // 1st Trigger: 2;6
+ inputs.add(2);
+ inputs.add(1);
+ inputs.add(2);
+ // 2nd Trigger: 1;4
+ inputs.add(2);
+ inputs.add(1);
+ // Final: 1,2
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(2);
+ expected.add(6);
+ expected.add(4);
+ expected.add(1);
+ expected.add(2);
+ expected.add(1);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
+ myReduceFunction, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, distributedTriggers, evictions, triggers);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+ /**
+ * Test for combination of centralized trigger and distributed trigger at
+ * the same time
+ */
+ @Test
+ public void testGroupedWindowInvokableCentralAndDistrTrigger() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(8));
+ triggers.add(new CountTriggerPolicy<Integer>(5));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // The active wrapper causes eviction even on (fake) elements which
+ // triggered, but does not belong to the group.
+ evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+ new TumblingEvictionPolicy<Integer>()));
+
+ LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+ distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ // local on 2 => 4
+ inputs.add(2);
+ inputs.add(1);
+ // local on 1 => 2
+ // and 1st Central: 2;2
+ // SUMS up to 2;2
+ inputs.add(2);
+ // local on 2 => 2
+ inputs.add(1);
+ inputs.add(2);
+ // 2nd Central: 1;2
+ inputs.add(2);
+ inputs.add(1);
+ // Final: 1,2
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(4);
+ expected.add(2);
+ expected.add(2);
+ expected.add(2);
+ expected.add(1);
+ expected.add(2);
+ expected.add(1);
+ expected.add(2);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
+ myReduceFunction, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, distributedTriggers, evictions, triggers);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/240e5f13/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
deleted file mode 100644
index 9d5744b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
-import org.junit.Test;
-
-public class GroupedWindowingInvokableTest {
-
- /**
- * Test for not active distributed triggers with single field
- */
- @Test
- public void testGroupedWindowingInvokableDistributedTriggerSimple() {
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(1);
- inputs.add(5);
- inputs.add(5);
- inputs.add(5);
- inputs.add(1);
- inputs.add(1);
- inputs.add(5);
- inputs.add(1);
- inputs.add(5);
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(15);
- expected.add(3);
- expected.add(3);
- expected.add(15);
-
- LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
- // Trigger on every 2nd element, but the first time after the 3rd
- triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
- LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
- // On every 2nd element, remove the oldest 2 elements, but the first
- // time after the 3rd element
- evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
- LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
-
- GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
- new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- }, new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) {
- return value;
- }
- }, triggers, evictions, centralTriggers);
-
- List<Integer> result = MockInvokable.createAndExecute(invokable, inputs);
-
- List<Integer> actual = new LinkedList<Integer>();
- for (Integer current : result) {
- actual.add(current);
- }
-
- assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(actual));
- assertEquals(expected.size(), actual.size());
- }
-
- /**
- * Test for non active distributed triggers with separated key field
- */
- @Test
- public void testGroupedWindowingInvokableDistributedTriggerComplex() {
- List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
- inputs2.add(new Tuple2<Integer, String>(1, "a"));
- inputs2.add(new Tuple2<Integer, String>(0, "b"));
- inputs2.add(new Tuple2<Integer, String>(2, "a"));
- inputs2.add(new Tuple2<Integer, String>(-1, "a"));
- inputs2.add(new Tuple2<Integer, String>(-2, "a"));
- inputs2.add(new Tuple2<Integer, String>(10, "a"));
- inputs2.add(new Tuple2<Integer, String>(2, "b"));
- inputs2.add(new Tuple2<Integer, String>(1, "a"));
-
- List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
- expected2.add(new Tuple2<Integer, String>(-1, "a"));
- expected2.add(new Tuple2<Integer, String>(-2, "a"));
- expected2.add(new Tuple2<Integer, String>(0, "b"));
-
- LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
- // Trigger on every 2nd element, but the first time after the 3rd
- triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
-
- LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
- // On every 2nd element, remove the oldest 2 elements, but the first
- // time after the 3rd element
- evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
-
- LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
-
- GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
- new ReduceFunction<Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
- Tuple2<Integer, String> value2) throws Exception {
- if (value1.f0 <= value2.f0) {
- return value1;
- } else {
- return value2;
- }
- }
- }, new TupleKeySelector<Tuple2<Integer, String>>(1), triggers, evictions,
- centralTriggers);
-
- List<Tuple2<Integer, String>> result = MockInvokable.createAndExecute(invokable2, inputs2);
-
- List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
- for (Tuple2<Integer, String> current : result) {
- actual2.add(current);
- }
-
- assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
- new HashSet<Tuple2<Integer, String>>(actual2));
- assertEquals(expected2.size(), actual2.size());
- }
-
- /**
- * Test for active centralized trigger
- */
- @Test
- public void testGroupedWindowingInvokableCentralActiveTrigger() {
-
- List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
- inputs.add(new Tuple2<Integer, String>(1, "a"));
- inputs.add(new Tuple2<Integer, String>(1, "b"));
- inputs.add(new Tuple2<Integer, String>(1, "c"));
- inputs.add(new Tuple2<Integer, String>(2, "a"));
- inputs.add(new Tuple2<Integer, String>(2, "b"));
- inputs.add(new Tuple2<Integer, String>(2, "c"));
- inputs.add(new Tuple2<Integer, String>(2, "b"));
- inputs.add(new Tuple2<Integer, String>(2, "a"));
- inputs.add(new Tuple2<Integer, String>(2, "c"));
- inputs.add(new Tuple2<Integer, String>(3, "c"));
- inputs.add(new Tuple2<Integer, String>(3, "a"));
- inputs.add(new Tuple2<Integer, String>(3, "b"));
- inputs.add(new Tuple2<Integer, String>(4, "a"));
- inputs.add(new Tuple2<Integer, String>(4, "b"));
- inputs.add(new Tuple2<Integer, String>(4, "c"));
- inputs.add(new Tuple2<Integer, String>(5, "c"));
- inputs.add(new Tuple2<Integer, String>(5, "a"));
- inputs.add(new Tuple2<Integer, String>(5, "b"));
- inputs.add(new Tuple2<Integer, String>(10, "b"));
- inputs.add(new Tuple2<Integer, String>(10, "a"));
- inputs.add(new Tuple2<Integer, String>(10, "c"));
- inputs.add(new Tuple2<Integer, String>(11, "a"));
- inputs.add(new Tuple2<Integer, String>(11, "a"));
- inputs.add(new Tuple2<Integer, String>(11, "c"));
- inputs.add(new Tuple2<Integer, String>(11, "c"));
- inputs.add(new Tuple2<Integer, String>(11, "b"));
- inputs.add(new Tuple2<Integer, String>(11, "b"));
-
- // Expected result:
- // For each group (a,b and c):
- // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
- // 12-12-5-10-32
-
- List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
- expected.add(new Tuple2<Integer, String>(12, "a"));
- expected.add(new Tuple2<Integer, String>(12, "b"));
- expected.add(new Tuple2<Integer, String>(12, "c"));
- expected.add(new Tuple2<Integer, String>(12, "a"));
- expected.add(new Tuple2<Integer, String>(12, "b"));
- expected.add(new Tuple2<Integer, String>(12, "c"));
- expected.add(new Tuple2<Integer, String>(5, "a"));
- expected.add(new Tuple2<Integer, String>(5, "b"));
- expected.add(new Tuple2<Integer, String>(5, "c"));
- expected.add(new Tuple2<Integer, String>(10, "a"));
- expected.add(new Tuple2<Integer, String>(10, "b"));
- expected.add(new Tuple2<Integer, String>(10, "c"));
- expected.add(new Tuple2<Integer, String>(32, "a"));
- expected.add(new Tuple2<Integer, String>(32, "b"));
- expected.add(new Tuple2<Integer, String>(32, "c"));
-
- TimeStamp<Tuple2<Integer, String>> myTimeStamp = new TimeStamp<Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Tuple2<Integer, String> value) {
- return value.f0;
- }
-
- @Override
- public long getStartTime() {
- return 1;
- }
- };
-
- ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
- Tuple2<Integer, String> value2) throws Exception {
- return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
- }
- };
-
- LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
- // Trigger every 2 time units but delay the first trigger by 2 (First
- // trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L));
-
- LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
- // Always delete all elements older then 4
- evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStamp));
-
- LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
-
- GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
- myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
- distributedTriggers, evictions, triggers);
-
- ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
- for (Tuple2<Integer, String> t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
- new HashSet<Tuple2<Integer, String>>(result));
- assertEquals(expected.size(), result.size());
- }
-
- /**
- * Test for multiple centralized trigger
- */
- @Test
- public void testGroupedWindowingInvokableMultipleCentralTrigger() {
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- triggers.add(new CountTriggerPolicy<Integer>(8));
- triggers.add(new CountTriggerPolicy<Integer>(5));
-
- LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
- // The active wrapper causes eviction even on (fake) elements which
- // triggered, but does not belong to the group.
- evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
- new TumblingEvictionPolicy<Integer>()));
-
- LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(2);
- inputs.add(2);
- inputs.add(2);
- inputs.add(1);
- // 1st Trigger: 2;6
- inputs.add(2);
- inputs.add(1);
- inputs.add(2);
- // 2nd Trigger: 1;4
- inputs.add(2);
- inputs.add(1);
- // Final: 1,2
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(2);
- expected.add(6);
- expected.add(4);
- expected.add(1);
- expected.add(2);
- expected.add(1);
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
- myReduceFunction, new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) {
- return value;
- }
- }, distributedTriggers, evictions, triggers);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
- assertEquals(expected.size(), result.size());
- }
-
- /**
- * Test for combination of centralized trigger and distributed trigger at
- * the same time
- */
- @Test
- public void testGroupedWindowingInvokableCentralAndDistrTrigger() {
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- triggers.add(new CountTriggerPolicy<Integer>(8));
- triggers.add(new CountTriggerPolicy<Integer>(5));
-
- LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
- // The active wrapper causes eviction even on (fake) elements which
- // triggered, but does not belong to the group.
- evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
- new TumblingEvictionPolicy<Integer>()));
-
- LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
- distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(2);
- inputs.add(2);
- // local on 2 => 4
- inputs.add(2);
- inputs.add(1);
- // local on 1 => 2
- // and 1st Central: 2;2
- // SUMS up to 2;2
- inputs.add(2);
- // local on 2 => 2
- inputs.add(1);
- inputs.add(2);
- // 2nd Central: 1;2
- inputs.add(2);
- inputs.add(1);
- // Final: 1,2
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(4);
- expected.add(2);
- expected.add(2);
- expected.add(2);
- expected.add(1);
- expected.add(2);
- expected.add(1);
- expected.add(2);
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
- myReduceFunction, new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer getKey(Integer value) {
- return value;
- }
- }, distributedTriggers, evictions, triggers);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
- assertEquals(expected.size(), result.size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/240e5f13/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
new file mode 100644
index 0000000..612da84
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.util.MockInvokable;
+import org.junit.Test;
+
+public class WindowInvokableTest {
+
+ /**
+ * Test case equal to {@link WindowReduceInvokableTest}
+ */
+ @Test
+ public void testWindowInvokableWithTimePolicy() {
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(3);
+ inputs.add(4);
+ inputs.add(5);
+ inputs.add(10);
+ inputs.add(11);
+ inputs.add(11);
+ // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+ // 12-12-5-10-32
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(12);
+ expected.add(12);
+ expected.add(5);
+ expected.add(10);
+ expected.add(32);
+
+ TimeStamp<Integer> myTimeStamp = new TimeStamp<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 1;
+ }
+ };
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger every 2 time units but delay the first trigger by 2 (First
+ // trigger after 4, then every 2)
+ triggers.add(new TimeTriggerPolicy<Integer>(2L, myTimeStamp, 2L));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // Always delete all elements older then 4
+ evictions.add(new TimeEvictionPolicy<Integer>(4L, myTimeStamp));
+
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(expected, result);
+ }
+
+ /**
+ * Test case equal to {@link BatchReduceTest}
+ */
+ @Test
+ public void testWindowInvokableWithCountPolicy() {
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ for (Integer i = 1; i <= 10; i++) {
+ inputs.add(i);
+ }
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ /*
+ * The following setup reassembles the batch size 3 and the slide size 2
+ * of the BatchReduceInvokable.
+ */
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(6);
+ expected.add(12);
+ expected.add(18);
+ expected.add(24);
+ expected.add(19);
+ List<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+ assertEquals(expected, result);
+
+ /*
+ * Begin test part 2
+ */
+
+ List<Integer> inputs2 = new ArrayList<Integer>();
+ inputs2.add(1);
+ inputs2.add(2);
+ inputs2.add(-5); // changed this value to make sure it is excluded from
+ // the result
+ inputs2.add(-3);
+ inputs2.add(-4);
+
+ myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ if (value1 <= value2) {
+ return value1;
+ } else {
+ return value2;
+ }
+ };
+ };
+
+ /*
+ * The following setup reassembles the batch size 2 and the slide size 3
+ * of the BatchReduceInvokable.
+ */
+ triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger after every 3rd element, but the first time after the 2nd
+ triggers.add(new CountTriggerPolicy<Integer>(3, 1));
+
+ evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // On every 3rd element, remove the oldest 3 elements, but the first
+ // time after on the 5th element
+ evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
+
+ WindowInvokable<Integer, Integer> invokable2 = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ List<Integer> expected2 = new ArrayList<Integer>();
+ expected2.add(1);
+ expected2.add(-4);
+
+ result = new ArrayList<Integer>();
+ for (Integer t : MockInvokable.createAndExecute(invokable2, inputs2)) {
+ result.add(t);
+ }
+
+ assertEquals(expected2, result);
+
+ }
+
+ @Test
+ public void testWindowInvokableWithMultiplePolicies() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(2));
+ triggers.add(new CountTriggerPolicy<Integer>(3));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2));
+ evictions.add(new CountEvictionPolicy<Integer>(3, 3));
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ for (Integer i = 1; i <= 10; i++) {
+ inputs.add(i);
+ }
+ /**
+ * <code>
+ * VAL: 1,2,3,4,5,6,7,8,9,10
+ * TR1: | | | | |
+ * TR2: | | |
+ * EV1: 2 2 2 2 2
+ * EV2: 3 3 3
+ * </code>
+ */
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(3);
+ expected.add(3);
+ expected.add(4);
+ expected.add(11);
+ expected.add(15);
+ expected.add(9);
+ expected.add(10);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(expected, result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/240e5f13/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
deleted file mode 100644
index 4c6b187..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.junit.Test;
-
-public class WindowingInvokableTest {
-
- /**
- * Test case equal to {@link WindowReduceInvokableTest}
- */
- @Test
- public void testWindowingInvokableWithTimePolicy() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(2);
- inputs.add(2);
- inputs.add(3);
- inputs.add(4);
- inputs.add(5);
- inputs.add(10);
- inputs.add(11);
- inputs.add(11);
- // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
- // 12-12-5-10-32
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(12);
- expected.add(12);
- expected.add(5);
- expected.add(10);
- expected.add(32);
-
- TimeStamp<Integer> myTimeStamp = new TimeStamp<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Integer value) {
- return value;
- }
-
- @Override
- public long getStartTime() {
- return 1;
- }
- };
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- // Trigger every 2 time units but delay the first trigger by 2 (First
- // trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Integer>(2L, myTimeStamp, 2L));
-
- LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
- // Always delete all elements older then 4
- evictions.add(new TimeEvictionPolicy<Integer>(4L, myTimeStamp));
-
- WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(expected, result);
- }
-
- /**
- * Test case equal to {@link BatchReduceTest}
- */
- @Test
- public void testWindowingInvokableWithCountPolicy() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- /*
- * The following setup reassembles the batch size 3 and the slide size 2
- * of the BatchReduceInvokable.
- */
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- // Trigger on every 2nd element, but the first time after the 3rd
- triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
- LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
- // On every 2nd element, remove the oldest 2 elements, but the first
- // time after the 3rd element
- evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
- WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(6);
- expected.add(12);
- expected.add(18);
- expected.add(24);
- expected.add(19);
- List<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
- assertEquals(expected, result);
-
- /*
- * Begin test part 2
- */
-
- List<Integer> inputs2 = new ArrayList<Integer>();
- inputs2.add(1);
- inputs2.add(2);
- inputs2.add(-5); // changed this value to make sure it is excluded from
- // the result
- inputs2.add(-3);
- inputs2.add(-4);
-
- myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- if (value1 <= value2) {
- return value1;
- } else {
- return value2;
- }
- };
- };
-
- /*
- * The following setup reassembles the batch size 2 and the slide size 3
- * of the BatchReduceInvokable.
- */
- triggers = new LinkedList<TriggerPolicy<Integer>>();
- // Trigger after every 3rd element, but the first time after the 2nd
- triggers.add(new CountTriggerPolicy<Integer>(3, 1));
-
- evictions = new LinkedList<EvictionPolicy<Integer>>();
- // On every 3rd element, remove the oldest 3 elements, but the first
- // time after on the 5th element
- evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
-
- WindowInvokable<Integer, Integer> invokable2 = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- List<Integer> expected2 = new ArrayList<Integer>();
- expected2.add(1);
- expected2.add(-4);
-
- result = new ArrayList<Integer>();
- for (Integer t : MockInvokable.createAndExecute(invokable2, inputs2)) {
- result.add(t);
- }
-
- assertEquals(expected2, result);
-
- }
-
- @Test
- public void testWindowingInvokableWithMultiplePolicies() {
- LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
- triggers.add(new CountTriggerPolicy<Integer>(2));
- triggers.add(new CountTriggerPolicy<Integer>(3));
-
- LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
- evictions.add(new CountEvictionPolicy<Integer>(2, 2));
- evictions.add(new CountEvictionPolicy<Integer>(3, 3));
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
- /**
- * <code>
- * VAL: 1,2,3,4,5,6,7,8,9,10
- * TR1: | | | | |
- * TR2: | | |
- * EV1: 2 2 2 2 2
- * EV2: 3 3 3
- * </code>
- */
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(3);
- expected.add(3);
- expected.add(4);
- expected.add(11);
- expected.add(15);
- expected.add(9);
- expected.add(10);
-
- ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- };
-
- WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
- myReduceFunction, triggers, evictions);
-
- ArrayList<Integer> result = new ArrayList<Integer>();
- for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t);
- }
-
- assertEquals(expected, result);
- }
-
-}
[02/34] incubator-flink git commit: [streaming] Created tumbling
eviction policy. This policy evicts always all tuples after a trigger
occurred. It can be used to prevent doubled computation between trigger and
eviction policies whenever tumbling windows
Posted by mb...@apache.org.
[streaming] Created tumbling eviction policy. This policy evicts always all tuples after a trigger occurred. It can be used to prevent doubled computation between trigger and eviction policies whenever tumbling windows are used. (includes test case)
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/fb4a8396
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/fb4a8396
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/fb4a8396
Branch: refs/heads/master
Commit: fb4a8396b113ee2e6b2543614f6210c44e0587d9
Parents: a6cc8af
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:49:16 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../policy/TumblingEvictionPolicy.java | 85 ++++++++++++++++++++
.../policy/TumblingEvictionPolicyTest.java | 41 ++++++++++
2 files changed, 126 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fb4a8396/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
new file mode 100644
index 0000000..dc7f5b0
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This eviction policy deletes all elements from the buffer in case a trigger
+ * occurred. Therefore, it is the default eviction policy to be used for any
+ * tumbling window.
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it clearing the buffer even on
+ * fake elements.
+ *
+ * @param <DATA>
+ * The type of the data points which is handled by this policy
+ */
+public class TumblingEvictionPolicy<DATA> implements EvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -4018019069267281155L;
+
+ /**
+ * Counter for the current number of elements in the buffer
+ */
+ private int counter = 0;
+
+ /**
+ * This is the default constructor providing no special functionality. This
+ * eviction policy deletes all elements from the buffer in case a trigger
+ * occurred. Therefore, it is the default eviction policy to be used for any
+ * tumbling window.
+ */
+ public TumblingEvictionPolicy() {
+ // default constructor, no further logic needed
+ }
+
+ /**
+ * This constructor allows to set a custom start value for the element
+ * counter.
+ *
+ * This eviction policy deletes all elements from the buffer in case a
+ * trigger occurred. Therefore, it is the default eviction policy to be used
+ * for any tumbling window.
+ *
+ * @param startValue
+ * A start value for the element counter
+ */
+ public TumblingEvictionPolicy(int startValue) {
+ this.counter = startValue;
+ }
+
+ /**
+ * Deletes all elements from the buffer in case the trigger occurred.
+ */
+ @Override
+ public int notifyEviction(Object datapoint, boolean triggered, int bufferSize) {
+ if (triggered) {
+ // The current data point will be part of the next window!
+ // Therefore the counter needs to be set to one already.
+ int tmpCounter = counter;
+ counter = 1;
+ return tmpCounter;
+ } else {
+ counter++;
+ return 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/fb4a8396/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
new file mode 100644
index 0000000..8cb03a7
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicyTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class TumblingEvictionPolicyTest {
+
+ @Test
+ public void testTumblingEviction() {
+ EvictionPolicy<Integer> policy = new TumblingEvictionPolicy<Integer>();
+
+ int counter = 0;
+
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < i; j++) {
+ assertEquals(0, policy.notifyEviction(0, false, counter++));
+ }
+ assertEquals(counter, policy.notifyEviction(0, true, counter));
+ counter = 1;
+ }
+ }
+
+}
[31/34] incubator-flink git commit: [streaming] Introduced central
eviction policies in the grouped window invokable
Posted by mb...@apache.org.
[streaming] Introduced central eviction policies in the grouped window invokable
[streaming] Extended test cases for grouped window invokable to cover usage of central eviction policies
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/b082af06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/b082af06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/b082af06
Branch: refs/heads/master
Commit: b082af06549f639d910173cac9589d23778cf350
Parents: 240e5f1
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Tue Nov 25 14:55:52 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:10 2014 +0100
----------------------------------------------------------------------
.../operator/GroupedWindowInvokable.java | 223 ++++++++++++++++---
.../api/invokable/operator/WindowInvokable.java | 12 +
.../operator/GroupedWindowInvokableTest.java | 202 ++++++++++++++---
3 files changed, 382 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b082af06/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
index 844a488..29e154b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.invokable.operator;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
+import java.util.NoSuchElementException;
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.GroupReduceFunction;
@@ -28,6 +29,7 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
@@ -43,21 +45,21 @@ import org.slf4j.LoggerFactory;
* versions. It is additionally aware of the creation of windows per group.
*
* A {@link KeySelector} is used to specify the key position or key extraction.
- * The {@link ReduceFunction} will be executed on each group separately. Trigger
- * policies might either be centralized or distributed. Eviction policies are
- * always distributed. A distributed policy have to be a
- * {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it will
- * be cloned to have separated instances for each group. At the startup time the
- * distributed policies will be stored as sample, and only clones of them will
- * be used to maintain the groups. Therefore, each group starts with the initial
- * policy states.
+ * The {@link ReduceFunction} will be executed on each group separately.
+ * Policies might either be centralized or distributed. It is not possible to
+ * use central and distributed eviction policies at the same time. A distributed
+ * policy have to be a {@link CloneableTriggerPolicy} or
+ * {@link CloneableEvictionPolicy} as it will be cloned to have separated
+ * instances for each group. At the startup time the distributed policies will
+ * be stored as sample, and only clones of them will be used to maintain the
+ * groups. Therefore, each group starts with the initial policy states.
*
* While a distributed policy only gets notified with the elements belonging to
* the respective group, a centralized policy get notified with all arriving
* elements. When a centralized trigger occurred, all groups get triggered. This
* is done by submitting the element which caused the trigger as real element to
* the groups it belongs to and as fake element to all other groups. Within the
- * groups the element might be further processed, causing more triggered,
+ * groups the element might be further processed, causing more triggers,
* prenotifications of active distributed policies and evictions like usual.
*
* Central policies can be instance of {@link ActiveTriggerPolicy} and also
@@ -83,23 +85,28 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
private Configuration parameters;
private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ private LinkedList<ActiveEvictionPolicy<IN>> activeCentralEvictionPolicies;
+ private LinkedList<EvictionPolicy<IN>> centralEvictionPolicies;
private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
private Map<Object, WindowInvokable<IN, OUT>> windowingGroups = new HashMap<Object, WindowInvokable<IN, OUT>>();
private LinkedList<Thread> activePolicyThreads = new LinkedList<Thread>();
private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-
+ private LinkedList<WindowInvokable<IN, OUT>> deleteOrderForCentralEviction;
+
/**
* This constructor creates an instance of the grouped windowing invokable.
+ *
* A {@link KeySelector} is used to specify the key position or key
* extraction. The {@link ReduceFunction} will be executed on each group
- * separately. Trigger policies might either be centralized or distributed.
- * Eviction policies are always distributed. A distributed policy have to be
- * a {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it
- * will be cloned to have separated instances for each group. At the startup
- * time the distributed policies will be stored as sample, and only clones
- * of them will be used to maintain the groups. Therefore, each group starts
- * with the initial policy states.
+ * separately. Policies might either be centralized or distributed. It is
+ * not possible to use central and distributed eviction policies at the same
+ * time. A distributed policy have to be a {@link CloneableTriggerPolicy} or
+ * {@link CloneableEvictionPolicy} as it will be cloned to have separated
+ * instances for each group. At the startup time the distributed policies
+ * will be stored as sample, and only clones of them will be used to
+ * maintain the groups. Therefore, each group starts with the initial policy
+ * states.
*
* While a distributed policy only gets notified with the elements belonging
* to the respective group, a centralized policy get notified with all
@@ -107,7 +114,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
* triggered. This is done by submitting the element which caused the
* trigger as real element to the groups it belongs to and as fake element
* to all other groups. Within the groups the element might be further
- * processed, causing more triggered, prenotifications of active distributed
+ * processed, causing more triggers, prenotifications of active distributed
* policies and evictions like usual.
*
* Central policies can be instance of {@link ActiveTriggerPolicy} and also
@@ -127,33 +134,88 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
* within each group.
* @param distributedEvictionPolicies
* Eviction policies to be distributed and maintained
- * individually within each group. There are no central eviction
- * policies because there is no central element buffer but only a
- * buffer per group. Therefore evictions might always be done per
- * group.
+ * individually within each group. Note that there cannot be
+ * both, central and distributed eviction policies at the same
+ * time.
* @param centralTriggerPolicies
* Trigger policies which will only exist once at a central
* place. In case a central policy triggers, it will cause all
* groups to be emitted. (Remark: Empty groups cannot be emitted.
* If only one element is contained a group, this element itself
* is returned as aggregated result.)
+ * @param centralEvictionPolicies
+ * Eviction which will only exist once at a central place. Note
+ * that there cannot be both, central and distributed eviction
+ * policies at the same time. The central eviction policy will
+ * work on an simulated element buffer containing all elements no
+ * matter which group they belong to.
*/
public GroupedWindowInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
- LinkedList<TriggerPolicy<IN>> centralTriggerPolicies) {
+ LinkedList<TriggerPolicy<IN>> centralTriggerPolicies,
+ LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
super(userFunction);
+
+ // check that not both, central and distributed eviction, is used at the
+ // same time.
+ if (centralEvictionPolicies != null && distributedEvictionPolicies != null
+ && !centralEvictionPolicies.isEmpty() && !distributedEvictionPolicies.isEmpty()) {
+ throw new UnsupportedOperationException(
+ "You can only use either central or distributed eviction policies but not both at the same time.");
+ }
+
+ // Check that there is at least one trigger and one eviction policy
+ if ((centralEvictionPolicies == null || centralEvictionPolicies.isEmpty())
+ && (distributedEvictionPolicies == null || distributedEvictionPolicies.isEmpty())) {
+ throw new UnsupportedOperationException(
+ "You have to define at least one eviction policy");
+ }
+ if ((centralTriggerPolicies == null || centralTriggerPolicies.isEmpty())
+ && (distributedTriggerPolicies == null || distributedTriggerPolicies.isEmpty())) {
+ throw new UnsupportedOperationException(
+ "You have to define at least one trigger policy");
+ }
+
this.keySelector = keySelector;
+
+ // handle the triggers
this.centralTriggerPolicies = centralTriggerPolicies;
this.distributedTriggerPolicies = distributedTriggerPolicies;
- this.distributedEvictionPolicies = distributedEvictionPolicies;
-
for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
if (trigger instanceof ActiveTriggerPolicy) {
this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
}
}
+
+ // handle the evictions
+ if (distributedEvictionPolicies != null && !distributedEvictionPolicies.isEmpty()) {
+ this.distributedEvictionPolicies = distributedEvictionPolicies;
+ } else { // (centralEvictionPolicies!=null&&!centralEvictionPolicies.isEmpty())
+ this.centralEvictionPolicies = centralEvictionPolicies;
+ this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+ for (EvictionPolicy<IN> eviction : centralEvictionPolicies) {
+ if (eviction instanceof ActiveEvictionPolicy) {
+ this.activeCentralEvictionPolicies.add((ActiveEvictionPolicy<IN>) eviction);
+ }
+ }
+ this.deleteOrderForCentralEviction = new LinkedList<WindowInvokable<IN, OUT>>();
+ }
+
+ }
+
+ /**
+ * Same as
+ * {@link GroupedWindowInvokable#GroupedWindowInvokable(Function, KeySelector, LinkedList, LinkedList, LinkedList, LinkedList)}
+ * but using always distributed eviction only.
+ */
+ public GroupedWindowInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
+ LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
+ LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
+ LinkedList<TriggerPolicy<IN>> centralTriggerPolicies) {
+ this(userFunction, keySelector, distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, null);
}
@Override
@@ -165,8 +227,8 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
// Continuously run
while (reuse != null) {
- WindowInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector
- .getKey(reuse.getObject()));
+ WindowInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector.getKey(reuse
+ .getObject()));
if (groupInvokable == null) {
groupInvokable = makeNewGroup(reuse);
}
@@ -175,6 +237,13 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
Object[] result = trigger.preNotifyTrigger(reuse.getObject());
for (Object in : result) {
+
+ // If central eviction is used, handle it here
+ if (activeCentralEvictionPolicies!=null) {
+ evictElements(centralActiveEviction(in));
+ }
+
+ // process in groups
for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(in, trigger);
}
@@ -189,9 +258,18 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
}
if (currentTriggerPolicies.isEmpty()) {
+
// only add the element to its group
groupInvokable.processRealElement(reuse.getObject());
+
+ // If central eviction is used, handle it here
+ if (centralEvictionPolicies!=null) {
+ evictElements(centralEviction(reuse.getObject(), false));
+ deleteOrderForCentralEviction.add(groupInvokable);
+ }
+
} else {
+
// call user function for all groups
for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
if (group == groupInvokable) {
@@ -203,6 +281,12 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
group.externalTriggerFakeElement(reuse.getObject(), currentTriggerPolicies);
}
}
+
+ // If central eviction is used, handle it here
+ if (centralEvictionPolicies!=null) {
+ evictElements(centralEviction(reuse.getObject(), true));
+ deleteOrderForCentralEviction.add(groupInvokable);
+ }
}
// clear current trigger list
@@ -228,8 +312,8 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
/**
* This method creates a new group. The method gets called in case an
* element arrives which has a key which was not seen before. The method
- * created a nested {@link WindowInvokable} and therefore created clones
- * of all distributed trigger and eviction policies.
+ * created a nested {@link WindowInvokable} and therefore created clones of
+ * all distributed trigger and eviction policies.
*
* @param element
* The element which leads to the generation of a new group
@@ -299,6 +383,80 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
};
/**
+ * This method is used to notify central eviction policies with a real
+ * element.
+ *
+ * @param input
+ * the real element to notify the eviction policy.
+ * @param triggered
+ * whether a central trigger occurred or not.
+ * @return The number of elements to be deleted from the buffer.
+ */
+ private int centralEviction(IN input, boolean triggered) {
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (EvictionPolicy<IN> evictionPolicy : centralEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEviction(input, triggered,
+ deleteOrderForCentralEviction.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+ return currentMaxEviction;
+ }
+
+ /**
+ * This method is used to notify active central eviction policies with a
+ * fake element.
+ *
+ * @param input
+ * the fake element to notify the active central eviction
+ * policies.
+ * @return The number of elements to be deleted from the buffer.
+ */
+ private int centralActiveEviction(Object input) {
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeCentralEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input,
+ deleteOrderForCentralEviction.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+ return currentMaxEviction;
+ }
+
+ /**
+ * This method is used in central eviction to delete a given number of
+ * elements from the buffer.
+ *
+ * @param numToEvict
+ * number of elements to delete from the virtual central element
+ * buffer.
+ */
+ private void evictElements(int numToEvict) {
+ for (; numToEvict > 0; numToEvict--) {
+ deleteOrderForCentralEviction.getFirst().evictFirst();
+ try {
+ deleteOrderForCentralEviction.removeFirst();
+ } catch (NoSuchElementException e) {
+ // when buffer is empty, ignore exception and stop deleting
+ break;
+ }
+
+ }
+ }
+
+ /**
* This callback class allows to handle the the callbacks done by threads
* defined in active trigger policies
*
@@ -313,6 +471,13 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
@Override
public void sendFakeElement(Object datapoint) {
+
+ // If central eviction is used, handle it here
+ if (centralEvictionPolicies!=null) {
+ evictElements(centralActiveEviction(datapoint));
+ }
+
+ // handle element in groups
for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(datapoint, policy);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b082af06/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
index 953759c..740fa4f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -339,6 +339,18 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
buffer.add(input);
}
+
+ /**
+ * This method removes the first element from the element buffer. It is used
+ * to provide central evictions in {@link GroupedWindowInvokable}
+ */
+ protected synchronized void evictFirst() {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // ignore exception
+ }
+ }
/**
* This method does the final reduce at the end of the stream and emits the
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b082af06/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
index 9d10166..5ac5529 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.api.invokable.operator;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import java.util.ArrayList;
import java.util.HashSet;
@@ -33,6 +34,7 @@ import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
@@ -44,6 +46,101 @@ import org.junit.Test;
public class GroupedWindowInvokableTest {
/**
+ * Tests that illegal arguments result in failure. The following cases are
+ * tested: 1) having no trigger 2) having no eviction 3) having neither
+ * eviction nor trigger 4) having both, central and distributed eviction.
+ */
+ @Test
+ public void testGroupedWindowInvokableFailTest() {
+
+ // create dummy reduce function
+ ReduceFunction<Object> userFunction = new ReduceFunction<Object>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object reduce(Object value1, Object value2) throws Exception {
+ return null;
+ }
+ };
+
+ // create dummy keySelector
+ KeySelector<Object, Object> keySelector = new KeySelector<Object, Object>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Object getKey(Object value) throws Exception {
+ return null;
+ }
+ };
+
+ // create policy lists
+ LinkedList<CloneableEvictionPolicy<Object>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<Object>>();
+ LinkedList<CloneableTriggerPolicy<Object>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<Object>>();
+ LinkedList<EvictionPolicy<Object>> centralEvictionPolicies = new LinkedList<EvictionPolicy<Object>>();
+ LinkedList<TriggerPolicy<Object>> centralTriggerPolicies = new LinkedList<TriggerPolicy<Object>>();
+
+ // empty trigger and policy lists should fail
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (1)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // null for trigger and policy lists should fail
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector, null, null, null,
+ null);
+ fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (2)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // empty eviction should still fail
+ centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+ distributedTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance without any eviction policy should cause an UnsupportedOperationException but didn't. (3)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // empty trigger should still fail
+ centralTriggerPolicies.clear();
+ distributedTriggerPolicies.clear();
+ centralEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance without any trigger policy should cause an UnsupportedOperationException but didn't. (4)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ // having both, central and distributed eviction, at the same time
+ // should fail
+ centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+ distributedEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
+ try {
+ new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
+ distributedTriggerPolicies, distributedEvictionPolicies,
+ centralTriggerPolicies, centralEvictionPolicies);
+ fail("Creating instance with central and distributed eviction should cause an UnsupportedOperationException but didn't. (4)");
+ } catch (UnsupportedOperationException e) {
+ // that's the expected case
+ }
+
+ }
+
+ /**
* Test for not active distributed triggers with single field
*/
@Test
@@ -60,12 +157,23 @@ public class GroupedWindowInvokableTest {
inputs.add(1);
inputs.add(5);
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(15);
- expected.add(3);
- expected.add(3);
- expected.add(15);
-
+ List<Integer> expectedDistributedEviction = new ArrayList<Integer>();
+ expectedDistributedEviction.add(15);
+ expectedDistributedEviction.add(3);
+ expectedDistributedEviction.add(3);
+ expectedDistributedEviction.add(15);
+
+ List<Integer> expectedCentralEviction = new ArrayList<Integer>();
+ expectedCentralEviction.add(2);
+ expectedCentralEviction.add(5);
+ expectedCentralEviction.add(15);
+ expectedCentralEviction.add(2);
+ expectedCentralEviction.add(5);
+ expectedCentralEviction.add(2);
+ expectedCentralEviction.add(5);
+ expectedCentralEviction.add(1);
+ expectedCentralEviction.add(5);
+
LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
// Trigger on every 2nd element, but the first time after the 3rd
triggers.add(new CountTriggerPolicy<Integer>(2, -1));
@@ -77,22 +185,26 @@ public class GroupedWindowInvokableTest {
LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
- GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
- new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
+ ReduceFunction<Integer> reduceFunction=new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- }, new KeySelector<Integer, Integer>() {
- private static final long serialVersionUID = 1L;
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ KeySelector<Integer, Integer> keySelector=new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
- @Override
- public Integer getKey(Integer value) {
- return value;
- }
- }, triggers, evictions, centralTriggers);
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ };
+
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
+ reduceFunction, keySelector, triggers, evictions, centralTriggers, null);
List<Integer> result = MockInvokable.createAndExecute(invokable, inputs);
@@ -101,8 +213,26 @@ public class GroupedWindowInvokableTest {
actual.add(current);
}
- assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(actual));
- assertEquals(expected.size(), actual.size());
+ assertEquals(new HashSet<Integer>(expectedDistributedEviction), new HashSet<Integer>(actual));
+ assertEquals(expectedDistributedEviction.size(), actual.size());
+
+ //Run test with central eviction
+ triggers.clear();
+ centralTriggers.add(new CountTriggerPolicy<Integer>(2, -1));
+ LinkedList<EvictionPolicy<Integer>> centralEvictions = new LinkedList<EvictionPolicy<Integer>>();
+ centralEvictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ invokable = new GroupedWindowInvokable<Integer, Integer>(
+ reduceFunction, keySelector, triggers, null, centralTriggers,centralEvictions);
+
+ result = MockInvokable.createAndExecute(invokable, inputs);
+ actual = new LinkedList<Integer>();
+ for (Integer current : result) {
+ actual.add(current);
+ }
+
+ assertEquals(new HashSet<Integer>(expectedCentralEviction), new HashSet<Integer>(actual));
+ assertEquals(expectedCentralEviction.size(), actual.size());
}
/**
@@ -150,7 +280,7 @@ public class GroupedWindowInvokableTest {
}
}
}, new TupleKeySelector<Tuple2<Integer, String>>(1), triggers, evictions,
- centralTriggers);
+ centralTriggers, null);
List<Tuple2<Integer, String>> result = MockInvokable.createAndExecute(invokable2, inputs2);
@@ -258,7 +388,7 @@ public class GroupedWindowInvokableTest {
GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
- distributedTriggers, evictions, triggers);
+ distributedTriggers, evictions, triggers, null);
ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
for (Tuple2<Integer, String> t : MockInvokable.createAndExecute(invokable, inputs)) {
@@ -268,6 +398,26 @@ public class GroupedWindowInvokableTest {
assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
new HashSet<Tuple2<Integer, String>>(result));
assertEquals(expected.size(), result.size());
+
+ // repeat the test with central eviction. The result should be the same.
+ triggers.clear();
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L));
+ evictions.clear();
+ LinkedList<EvictionPolicy<Tuple2<Integer, String>>> centralEvictions = new LinkedList<EvictionPolicy<Tuple2<Integer, String>>>();
+ centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStamp));
+
+ invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
+ distributedTriggers, evictions, triggers, centralEvictions);
+
+ result = new ArrayList<Tuple2<Integer, String>>();
+ for (Tuple2<Integer, String> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(result));
+ assertEquals(expected.size(), result.size());
}
/**
@@ -327,7 +477,7 @@ public class GroupedWindowInvokableTest {
public Integer getKey(Integer value) {
return value;
}
- }, distributedTriggers, evictions, triggers);
+ }, distributedTriggers, evictions, triggers, null);
ArrayList<Integer> result = new ArrayList<Integer>();
for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
@@ -403,7 +553,7 @@ public class GroupedWindowInvokableTest {
public Integer getKey(Integer value) {
return value;
}
- }, distributedTriggers, evictions, triggers);
+ }, distributedTriggers, evictions, triggers, null);
ArrayList<Integer> result = new ArrayList<Integer>();
for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
[22/34] incubator-flink git commit: [streaming] Window join api
rework to support functional style
Posted by mb...@apache.org.
[streaming] Window join api rework to support functional style
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/0ef399cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/0ef399cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/0ef399cf
Branch: refs/heads/master
Commit: 0ef399cf7460c131133c3a9bd5eb2bac45e12d1b
Parents: 2d73c3f
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Nov 24 22:57:42 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../api/datastream/ConnectedDataStream.java | 43 +---
.../streaming/api/datastream/DataStream.java | 145 +++--------
.../api/datastream/StreamJoinOperator.java | 252 +++++++++++++++++++
.../api/function/co/JoinWindowFunction.java | 16 +-
.../streaming/api/WindowCrossJoinTest.java | 8 +-
.../examples/window/join/WindowJoin.java | 5 +-
6 files changed, 295 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/0ef399cf/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index 6556550..d2b8332 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -33,7 +33,6 @@ import org.apache.flink.streaming.api.function.co.CoMapFunction;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.function.co.CoWindowFunction;
import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
-import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
import org.apache.flink.streaming.api.function.co.RichCoMapFunction;
import org.apache.flink.streaming.api.function.co.RichCoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
@@ -563,47 +562,7 @@ public class ConnectedDataStream<IN1, IN2> {
timestamp1, timestamp2);
}
- SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> windowJoin(long windowSize, long slideInterval,
- String fieldIn1, String fieldIn2) {
-
- return windowJoin(windowSize, slideInterval, new DefaultTimeStamp<IN1>(),
- new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
- }
-
- SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> windowJoin(long windowSize, long slideInterval,
- int fieldIn1, int fieldIn2) {
-
- return windowJoin(windowSize, slideInterval, new DefaultTimeStamp<IN1>(),
- new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
- }
-
- SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> windowJoin(long windowSize, long slideInterval,
- TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2, int fieldIn1, int fieldIn2) {
-
- dataStream1 = dataStream1.groupBy(fieldIn1);
- dataStream2 = dataStream2.groupBy(fieldIn2);
-
- JoinWindowFunction<IN1, IN2> joinWindowFunction = new JoinWindowFunction<IN1, IN2>(
- dataStream1.getOutputType(), dataStream2.getOutputType(), fieldIn1, fieldIn2);
-
- return addGeneralWindowJoin(joinWindowFunction, windowSize, slideInterval, timestamp1,
- timestamp2);
- }
-
- SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> windowJoin(long windowSize, long slideInterval,
- TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2, String fieldIn1, String fieldIn2) {
-
- dataStream1 = dataStream1.groupBy(fieldIn1);
- dataStream2 = dataStream2.groupBy(fieldIn2);
-
- JoinWindowFunction<IN1, IN2> joinWindowFunction = new JoinWindowFunction<IN1, IN2>(
- dataStream1.getOutputType(), dataStream2.getOutputType(), fieldIn1, fieldIn2);
-
- return addGeneralWindowJoin(joinWindowFunction, windowSize, slideInterval, timestamp1,
- timestamp2);
- }
-
- private SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> addGeneralWindowJoin(
+ protected SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> addGeneralWindowJoin(
CoWindowFunction<IN1, IN2, Tuple2<IN1, IN2>> coWindowFunction, long windowSize,
long slideInterval, TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/0ef399cf/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 49981a5..13a7ba1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -240,9 +240,9 @@ public class DataStream<OUT> {
* @return The grouped {@link DataStream}
*/
public GroupedDataStream<OUT> groupBy(int... fields) {
-
+
return groupBy(FieldsKeySelector.getSelector(getOutputType(), fields));
-
+
}
/**
@@ -258,9 +258,9 @@ public class DataStream<OUT> {
* @return The grouped {@link DataStream}
**/
public GroupedDataStream<OUT> groupBy(String... fields) {
-
+
return groupBy(new PojoKeySelector<OUT>(getOutputType(), fields));
-
+
}
/**
@@ -286,7 +286,7 @@ public class DataStream<OUT> {
* @return The DataStream with fields partitioning set.
*/
public DataStream<OUT> partitionBy(int... fields) {
-
+
return setConnectionType(new FieldsPartitioner<OUT>(FieldsKeySelector.getSelector(
getOutputType(), fields)));
}
@@ -300,7 +300,7 @@ public class DataStream<OUT> {
* @return The DataStream with fields partitioning set.
*/
public DataStream<OUT> partitionBy(String... fields) {
-
+
return setConnectionType(new FieldsPartitioner<OUT>(new PojoKeySelector<OUT>(
getOutputType(), fields)));
}
@@ -467,7 +467,7 @@ public class DataStream<OUT> {
public SingleOutputStreamOperator<OUT, ?> filter(FilterFunction<OUT> filter) {
FunctionTypeWrapper<OUT> typeWrapper = new FunctionTypeWrapper<OUT>(filter,
FilterFunction.class, 0);
-
+
return addFunction("filter", filter, typeWrapper, typeWrapper, new FilterInvokable<OUT>(
filter));
}
@@ -496,6 +496,27 @@ public class DataStream<OUT> {
}
/**
+ * Initiates a temporal Join transformation. <br/>
+ * A temporal Join transformation joins the elements of two
+ * {@link DataStream}s on key equality over a specified time window.</br>
+ *
+ * This method returns a {@link StreamJoinOperator} on which the
+ * {@link StreamJoinOperator#onWindow} should be called to define the
+ * window, and then the {@link StreamJoinOperator.JoinWindow#where} and
+ * {@link StreamJoinOperator.JoinPredicate#equalTo} can be used to define
+ * the join keys.
+ *
+ * @param other
+ * The other DataStream with which this DataStream is joined.
+ * @return A {@link StreamJoinOperator} to continue the definition of the
+ * Join transformation.
+ *
+ */
+ public <IN2> StreamJoinOperator<OUT, IN2> join(DataStream<IN2> dataStreamToJoin) {
+ return new StreamJoinOperator<OUT, IN2>(this, dataStreamToJoin);
+ }
+
+ /**
* Applies an aggregation that sums the data stream at the given position.
*
* @param positionToSum
@@ -790,110 +811,6 @@ public class DataStream<OUT> {
}
/**
- * Creates a join of a data stream based on the given positions.
- *
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval, int fieldIn1,
- int fieldIn2) {
- return this.windowJoin(dataStreamToJoin, windowSize, slideInterval,
- new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
- }
-
- /**
- * Creates a join of a data stream based on the given positions.
- *
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval, String fieldIn1,
- String fieldIn2) {
- return this.windowJoin(dataStreamToJoin, windowSize, slideInterval,
- new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
- }
-
- /**
- * Creates a join of a data stream based on the given positions.
- *
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param timestamp1
- * User defined time stamps for the first input.
- * @param timestamp2
- * User defined time stamps for the second input.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval,
- TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2, int fieldIn1, int fieldIn2) {
- return this.connect(dataStreamToJoin).windowJoin(windowSize, slideInterval, timestamp1,
- timestamp2, fieldIn1, fieldIn2);
- }
-
- /**
- * Creates a join of a data stream based on the given positions.
- *
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param timestamp1
- * User defined time stamps for the first input.
- * @param timestamp2
- * User defined time stamps for the second input.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval,
- TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2, String fieldIn1, String fieldIn2) {
- return this.connect(dataStreamToJoin).windowJoin(windowSize, slideInterval, timestamp1,
- timestamp2, fieldIn1, fieldIn2);
- }
-
- /**
* Writes a DataStream to the standard output stream (stdout). For each
* element of the DataStream the result of {@link Object#toString()} is
* written.
@@ -1208,12 +1125,12 @@ public class DataStream<OUT> {
}
protected SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
-
+
StreamReduceInvokable<OUT> invokable = new StreamReduceInvokable<OUT>(aggregate);
-
+
SingleOutputStreamOperator<OUT, ?> returnStream = addFunction("reduce", aggregate,
outTypeWrapper, outTypeWrapper, invokable);
-
+
return returnStream;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/0ef399cf/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
new file mode 100644
index 0000000..fb76d28
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
+import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.util.keys.FieldsKeySelector;
+import org.apache.flink.streaming.util.keys.PojoKeySelector;
+
+public class StreamJoinOperator<I1, I2> {
+
+ private final DataStream<I1> input1;
+ private final DataStream<I2> input2;
+
+ long windowSize;
+ long slideInterval;
+
+ TimeStamp<I1> timeStamp1;
+ TimeStamp<I2> timeStamp2;
+
+ public StreamJoinOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ if (input1 == null || input2 == null) {
+ throw new NullPointerException();
+ }
+ this.input1 = input1.copy();
+ this.input2 = input2.copy();
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public JoinWindow onWindow(long windowSize) {
+ return onWindow(windowSize, windowSize);
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param slideInterval
+ * The slide size of the window.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public JoinWindow onWindow(long windowSize, long slideInterval) {
+ return onWindow(windowSize, slideInterval, new DefaultTimeStamp<I1>(),
+ new DefaultTimeStamp<I2>());
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param slideInterval
+ * The slide size of the window.
+ * @param timeStamp1
+ * The timestamp used to extract time from the elements of the
+ * first data stream.
+ * @param timeStamp2
+ * The timestamp used to extract time from the elements of the
+ * second data stream.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public JoinWindow onWindow(long windowSize, long slideInterval, TimeStamp<I1> timeStamp1,
+ TimeStamp<I2> timeStamp2) {
+
+ this.windowSize = windowSize;
+ this.slideInterval = slideInterval;
+
+ this.timeStamp1 = timeStamp1;
+ this.timeStamp2 = timeStamp2;
+
+ return new JoinWindow();
+ }
+
+ public class JoinWindow {
+
+ private JoinWindow() {
+
+ }
+
+ /**
+ * Continues a temporal Join transformation. <br/>
+ * Defines the {@link Tuple} fields of the first join {@link DataStream}
+ * that should be used as join keys.<br/>
+ * <b>Note: Fields can only be selected as join keys on Tuple
+ * DataStreams.</b><br/>
+ *
+ * @param fields
+ * The indexes of the other Tuple fields of the first join
+ * DataStreams that should be used as keys.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public JoinPredicate where(int... fields) {
+ return new JoinPredicate(FieldsKeySelector.getSelector(input1.getOutputType(), fields));
+ }
+
+ /**
+ * Continues a temporal join transformation. <br/>
+ * Defines the fields of the first join {@link DataStream} that should
+ * be used as grouping keys. Fields are the names of member fields of
+ * the underlying type of the data stream.
+ *
+ * @param fields
+ * The fields of the first join DataStream that should be
+ * used as keys.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public JoinPredicate where(String... fields) {
+ return new JoinPredicate(new PojoKeySelector<I1>(input1.getOutputType(), fields));
+ }
+
+ /**
+ * Continues a temporal Join transformation and defines a
+ * {@link KeySelector} function for the first join {@link DataStream}
+ * .</br> The KeySelector function is called for each element of the
+ * first DataStream and extracts a single key value on which the
+ * DataStream is joined. </br>
+ *
+ * @param keySelector
+ * The KeySelector function which extracts the key values
+ * from the DataStream on which it is joined.
+ * @return An incomplete Join transformation. Call
+ * {@link JoinPredicate#equalTo} to continue the Join.
+ */
+ public <K> JoinPredicate where(KeySelector<I1, K> keySelector) {
+ return new JoinPredicate(keySelector);
+ }
+
+ // ----------------------------------------------------------------------------------------
+
+ }
+
+ /**
+ * Intermediate step of a temporal Join transformation. <br/>
+ * To continue the Join transformation, select the join key of the second
+ * input {@link DataStream} by calling {@link JoinPredicate#equalTo}
+ *
+ */
+ public class JoinPredicate {
+
+ private final KeySelector<I1, ?> keys1;
+
+ private JoinPredicate(KeySelector<I1, ?> keys1) {
+ this.keys1 = keys1;
+ }
+
+ /**
+ * Continues a temporal Join transformation and defines the
+ * {@link Tuple} fields of the second join {@link DataStream} that
+ * should be used as join keys.<br/>
+ * <b>Note: Fields can only be selected as join keys on Tuple
+ * DataStreams.</b><br/>
+ *
+ * The resulting operator wraps each pair of joining elements into a
+ * {@link Tuple2}, with the element of the first input being the first
+ * field of the tuple and the element of the second input being the
+ * second field of the tuple.
+ *
+ * @param fields
+ * The indexes of the Tuple fields of the second join
+ * DataStream that should be used as keys.
+ * @return The joined data stream.
+ */
+ public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(int... fields) {
+ return createJoinOperator(FieldsKeySelector.getSelector(input2.getOutputType(), fields));
+ }
+
+ /**
+ * Continues a temporal Join transformation and defines the fields of
+ * the second join {@link DataStream} that should be used as join keys.<br/>
+ *
+ * The resulting operator wraps each pair of joining elements into a
+ * {@link Tuple2}, with the element of the first input being the first
+ * field of the tuple and the element of the second input being the
+ * second field of the tuple.
+ *
+ * @param fields
+ * The fields of the second join DataStream that should be
+ * used as keys.
+ * @return The joined data stream.
+ */
+ public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(String... fields) {
+ return createJoinOperator(new PojoKeySelector<I2>(input2.getOutputType(), fields));
+ }
+
+ /**
+ * Continues a temporal Join transformation and defines a
+ * {@link KeySelector} function for the second join {@link DataStream}
+ * .</br> The KeySelector function is called for each element of the
+ * second DataStream and extracts a single key value on which the
+ * DataStream is joined. </br>
+ *
+ * The resulting operator wraps each pair of joining elements into a
+ * {@link Tuple2}, with the element of the first input being the first
+ * field of the tuple and the element of the second input being the
+ * second field of the tuple.
+ *
+ * @param keySelector
+ * The KeySelector function which extracts the key values
+ * from the second DataStream on which it is joined.
+ * @return The joined data stream.
+ */
+ public <K> SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(
+ KeySelector<I2, K> keySelector) {
+ return createJoinOperator(keySelector);
+ }
+
+ protected SingleOutputStreamOperator<Tuple2<I1, I2>, ?> createJoinOperator(
+ KeySelector<I2, ?> keys2) {
+
+ JoinWindowFunction<I1, I2> joinWindowFunction = new JoinWindowFunction<I1, I2>(keys1,
+ keys2);
+ return input1.connect(input2).addGeneralWindowJoin(joinWindowFunction, windowSize,
+ slideInterval, timeStamp1, timeStamp2);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/0ef399cf/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
index cb501c6..53e2657 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/co/JoinWindowFunction.java
@@ -20,11 +20,8 @@ package org.apache.flink.streaming.api.function.co;
import java.util.List;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.util.keys.FieldsKeySelector;
-import org.apache.flink.streaming.util.keys.PojoKeySelector;
import org.apache.flink.util.Collector;
public class JoinWindowFunction<IN1, IN2> implements CoWindowFunction<IN1, IN2, Tuple2<IN1, IN2>> {
@@ -36,16 +33,9 @@ public class JoinWindowFunction<IN1, IN2> implements CoWindowFunction<IN1, IN2,
public JoinWindowFunction() {
}
- public JoinWindowFunction(TypeInformation<IN1> inType1, TypeInformation<IN2> inType2,
- int positionIn1, int positionIn2) {
- keySelector1 = FieldsKeySelector.getSelector(inType1, positionIn1);
- keySelector2 = FieldsKeySelector.getSelector(inType2, positionIn2);
- }
-
- public JoinWindowFunction(TypeInformation<IN1> inType1, TypeInformation<IN2> inType2,
- String field1, String field2) {
- keySelector1 = new PojoKeySelector<IN1>(inType1, field1);
- keySelector2 = new PojoKeySelector<IN2>(inType2, field2);
+ public JoinWindowFunction(KeySelector<IN1, ?> keySelector1, KeySelector<IN2, ?> keySelector2) {
+ this.keySelector1 = keySelector1;
+ this.keySelector2 = keySelector2;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/0ef399cf/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
index 3a9eedd..3b719be 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -90,14 +90,14 @@ public class WindowCrossJoinTest {
DataStream<Tuple2<Integer, String>> inStream1 = env.fromCollection(in1);
DataStream<Integer> inStream2 = env.fromCollection(in2);
- inStream1.windowJoin(inStream2, 1000, 1000, new MyTimestamp1(), new MyTimestamp2(), 0, 0)
- .addSink(new JoinResultSink());
+ inStream1.join(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
+ .where(0).equalTo(0).addSink(new JoinResultSink());
inStream1.windowCross(inStream2, 1000, 1000, new MyTimestamp1(), new MyTimestamp2())
- .addSink(new CrossResultSink());
+ .addSink(new CrossResultSink());
env.executeTest(MEMORYSIZE);
-
+
assertEquals(joinExpectedResults, joinResults);
assertEquals(crossExpectedResults, crossResults);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/0ef399cf/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/window/join/WindowJoin.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/window/join/WindowJoin.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/window/join/WindowJoin.java
index 6d8656c..d5f921e 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/window/join/WindowJoin.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/window/join/WindowJoin.java
@@ -62,7 +62,10 @@ public class WindowJoin {
// apply a temporal join over the two stream based on the names over one
// second windows
DataStream<Tuple2<Tuple2<String, Integer>, Tuple2<String, Integer>>> joinedStream = grades
- .windowJoin(salaries, 1000, 1000, 0, 0);
+ .join(salaries)
+ .onWindow(1000)
+ .where(0)
+ .equalTo(0);
// emit result
if (fileOutput) {
[32/34] incubator-flink git commit: [streaming] Changed
GroupedWindowInvokable to always delete groups in case they have an empty
element buffer
Posted by mb...@apache.org.
[streaming] Changed GroupedWindowInvokable to always delete groups in case they have an empty element buffer
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/783aa4a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/783aa4a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/783aa4a8
Branch: refs/heads/master
Commit: 783aa4a86b6815a8cf2929d79334bebbe4fc4eb6
Parents: 4e046a9
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Wed Dec 3 12:30:44 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:47:18 2014 +0100
----------------------------------------------------------------------
.../operator/GroupedWindowInvokable.java | 33 +++++++++++++++++++-
.../api/invokable/operator/WindowInvokable.java | 11 +++++++
2 files changed, 43 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/783aa4a8/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
index 905b45f..ae16be0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.api.invokable.operator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -252,6 +253,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
// process in groups
for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(in, trigger);
+ checkForEmptyGroupBuffer(group);
}
}
}
@@ -267,6 +269,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
// only add the element to its group
groupInvokable.processRealElement(reuse.getObject());
+ checkForEmptyGroupBuffer(groupInvokable);
// If central eviction is used, handle it here
if (!centralEvictionPolicies.isEmpty()) {
@@ -286,6 +289,9 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
// policies
group.externalTriggerFakeElement(reuse.getObject(), currentTriggerPolicies);
}
+
+ //remove group in case it has an empty buffer
+ //checkForEmptyGroupBuffer(group);
}
// If central eviction is used, handle it here
@@ -450,8 +456,13 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
* buffer.
*/
private void evictElements(int numToEvict) {
+ HashSet<WindowInvokable<IN, OUT>> usedGroups=new HashSet<WindowInvokable<IN,OUT>>();
for (; numToEvict > 0; numToEvict--) {
- deleteOrderForCentralEviction.getFirst().evictFirst();
+ WindowInvokable<IN, OUT> currentGroup=deleteOrderForCentralEviction.getFirst();
+ //Do the eviction
+ currentGroup.evictFirst();
+ //Remember groups which possibly have an empty buffer after the eviction
+ usedGroups.add(currentGroup);
try {
deleteOrderForCentralEviction.removeFirst();
} catch (NoSuchElementException e) {
@@ -460,6 +471,25 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
}
}
+
+ //Remove groups with empty buffer
+ for (WindowInvokable<IN, OUT> group:usedGroups){
+ checkForEmptyGroupBuffer(group);
+ }
+ }
+
+ /**
+ * Checks if the element buffer of a given windowing group is empty. If so,
+ * the group will be deleted.
+ *
+ * @param group
+ * The windowing group to be checked and and removed in case its
+ * buffer is empty.
+ */
+ private void checkForEmptyGroupBuffer(WindowInvokable<IN, OUT> group) {
+ if (group.isBufferEmpty()) {
+ windowingGroups.remove(group);
+ }
}
/**
@@ -486,6 +516,7 @@ public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
// handle element in groups
for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(datapoint, policy);
+ checkForEmptyGroupBuffer(group);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/783aa4a8/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
index bbc1277..0e740bb 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -357,6 +357,17 @@ public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
}
/**
+ * This method returns whether the element buffer is empty or not. It is
+ * used to figure out if a group can be deleted or not when
+ * {@link GroupedWindowInvokable} is used.
+ *
+ * @return true in case the buffer is empty otherwise false.
+ */
+ protected boolean isBufferEmpty(){
+ return buffer.isEmpty();
+ }
+
+ /**
* This method does the final reduce at the end of the stream and emits the
* result.
*
[26/34] incubator-flink git commit: [streaming] Created several
examples for the usage of policy based windowing.
Posted by mb...@apache.org.
[streaming] Created several examples for the usage of policy based windowing.
[streaming] Fixed windowing examples by removing manual setting of source parallelism
[streaming] Extended streaming guide with a section about the new policy based windowing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/3d11d5ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/3d11d5ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/3d11d5ba
Branch: refs/heads/master
Commit: 3d11d5baea1513a819f189f199502fc3979336b8
Parents: 3f50683
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Tue Oct 28 11:14:33 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
docs/streaming_guide.md | 19 +++
.../examples/windowing/BasicExample.java | 76 ++++++++++++
.../examples/windowing/DeltaExtractExample.java | 78 ++++++++++++
.../windowing/MultiplePoliciesExample.java | 84 +++++++++++++
.../examples/windowing/SlidingExample.java | 101 ++++++++++++++++
.../windowing/TimeWindowingExample.java | 120 +++++++++++++++++++
6 files changed, 478 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3d11d5ba/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index 264140a..a8002ff 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -285,6 +285,25 @@ The transformation calls a `GroupReduceFunction` for each data batch or data win
dataStream.batch(1000, 100).reduceGroup(reducer);
~~~
+#### Policy based windowing
+The policy based windowing is an highly flexible way to specify your stream discretisation also called windowing semantics. Two types of policies are used for such a specification:
+
+1) `TriggerPolicy` This policy defines when to trigger the reduce UDF on the current window and emit the result. In the API it completes a window statement such as: `.window(..).every(...)`, while we pass the triggering policy within `every`. Several predefind policies are provided in the API, including delta-based, punctuation based, count-based and time-based policies. Anyhow, policies are in general UDFs and can implement any custom behaviour as well.
+
+2) `Eviction Policy` This policy defines the length of a window as a means of a predicate for evicting tuples when they are no longer needed. In the API this can be defined by the `.window(..)` operation on a stream. There are mostely the same predefined policy types provided as for trigger policies.
+
+For example, if we want a fife tuples long sliding window that triggers every second we can write the following:
+~~~java
+myStream.window(Count.of(5)).every(Time.of(1,TimeUnit.SECOND))
+~~~
+
+In addition to this, multiple policies can be added to a `LinkedList` and then be used in parallel by writing the following:
+~~~java
+myStream.window(ListOfTriggerPolicies,ListOfEvictionPolicies)
+~~~
+
+Especially when time based windowing is used, you may want to trigger not only when an element arrives but also in between. Active policies provide this functionality. The predefined time-based policies are already implemented in such an active way and can hold as an example in case you want to implement your own user defined active policy. Anyhow, as our time-based trigger and eviction policies can work with user defined `TimeStamp` implementations, this policies already cover most use cases.
+
### Co operators
Co operators allow the users to jointly transform two `DataStreams` of different types providing a simple way to jointly manipulate a shared state. It is designed to support joint stream transformations where merging is not appropriate due to different data types or the in cases when user needs explicit track of the datas origin.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3d11d5ba/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java
new file mode 100644
index 0000000..c0e768b
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.util.Collector;
+
+/**
+ * A minimal example as introduction to the policy based windowing
+ */
+public class BasicExample {
+
+ private static final int PARALLELISM = 1;
+
+ public static void main(String[] args) throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(PARALLELISM);
+
+ // This reduce function does a String concat.
+ ReduceFunction<String> reduceFunction = new ReduceFunction<String>() {
+
+ /**
+ * Auto generates version ID
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String reduce(String value1, String value2) throws Exception {
+ return value1 + "|" + value2;
+ }
+
+ };
+
+ DataStream<Tuple2<String, String[]>> stream = env.addSource(new BasicSource())
+ .window(Count.of(5)).every(Count.of(2)).reduce(reduceFunction);
+
+ stream.print();
+
+ env.execute();
+ }
+
+ public static class BasicSource implements SourceFunction<String> {
+
+ private static final long serialVersionUID = 1L;
+ String str = new String("streaming");
+
+ @Override
+ public void invoke(Collector<String> out) throws Exception {
+ // continuous emit
+ while (true) {
+ out.collect(str);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3d11d5ba/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
new file mode 100644
index 0000000..0622dbf
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/DeltaExtractExample.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.deltafunction.EuclideanDistance;
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromTuple;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.api.windowing.helper.Delta;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example gives an impression about how to use delta policies. It also
+ * shows how extractors can be used.
+ */
+public class DeltaExtractExample {
+
+ private static final int PARALLELISM = 1;
+
+ @SuppressWarnings({ "serial", "rawtypes", "unchecked" })
+ public static void main(String[] args) throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(PARALLELISM);
+
+ ReduceFunction<Tuple3<Double, Double, String>> concatStrings = new ReduceFunction<Tuple3<Double, Double, String>>() {
+ @Override
+ public Tuple3 reduce(Tuple3 value1, Tuple3 value2) throws Exception {
+ return new Tuple3(value1.f0, value2.f1, value1.f2 + "|" + value2.f2);
+ }
+ };
+
+ DataStream dstream = env
+ .addSource(new CountingSource())
+ .window(Delta.of(new EuclideanDistance(new FieldsFromTuple(0, 1)), new Tuple3(0d,
+ 0d, "foo"), 1.2)).every(Count.of(2)).reduce(concatStrings);
+
+ dstream.print();
+ env.execute();
+
+ }
+
+ @SuppressWarnings("serial")
+ private static class CountingSource implements SourceFunction<Tuple3<Double, Double, String>> {
+
+ private int counter = 0;
+
+ @Override
+ public void invoke(Collector<Tuple3<Double, Double, String>> collector) throws Exception {
+ while (true) {
+ if (counter > 9999) {
+ counter = 0;
+ }
+ collector.collect(new Tuple3<Double, Double, String>((double) counter,
+ (double) counter + 1, "V" + counter++));
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3d11d5ba/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
new file mode 100644
index 0000000..069693f
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.util.Collector;
+
+import java.util.LinkedList;
+
+/**
+ * This example uses count based tumbling windowing with multiple eviction
+ * policies at the same time.
+ */
+public class MultiplePoliciesExample {
+
+ private static final int PARALLELISM = 1;
+
+ public static void main(String[] args) throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(PARALLELISM);
+
+ LinkedList<TriggerPolicy<String>> policies = new LinkedList<TriggerPolicy<String>>();
+ policies.add(new CountTriggerPolicy<String>(5));
+ policies.add(new CountTriggerPolicy<String>(8));
+
+ // This reduce function does a String concat.
+ ReduceFunction<String> reducer = new ReduceFunction<String>() {
+
+ /**
+ * Auto generates version ID
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String reduce(String value1, String value2) throws Exception {
+ return value1 + "|" + value2;
+ }
+
+ };
+
+ DataStream<Tuple2<String, String[]>> stream = env.addSource(new BasicSource()).window(
+ policies, reducer);
+
+ stream.print();
+
+ env.execute();
+ }
+
+ public static class BasicSource implements SourceFunction<String> {
+
+ private static final long serialVersionUID = 1L;
+ String str = new String("streaming");
+
+ @Override
+ public void invoke(Collector<String> out) throws Exception {
+ // continuous emit
+ while (true) {
+ out.collect(str);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3d11d5ba/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
new file mode 100644
index 0000000..1768480
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example uses count based sliding windows to illustrate different
+ * possibilities for the realization of sliding windows. Take a look on the code
+ * which is commented out to see different setups.
+ */
+public class SlidingExample {
+
+ private static final int PARALLELISM = 1;
+
+ public static void main(String[] args) throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(PARALLELISM);
+
+ /*
+ * SIMPLE-EXAMPLE: Use this to always keep the newest 10 elements in the
+ * buffer Resulting windows will have an overlap of 5 elements
+ */
+ // TriggerPolicy<String> triggerPolicy=new
+ // CountTriggerPolicy<String>(5);
+ // EvictionPolicy<String> evictionPolicy=new
+ // CountEvictionPolicy<String>(10);
+
+ /*
+ * ADVANCED-EXAMPLE: Use this to have the last element of the last
+ * window as first element of the next window while the window size is
+ * always 5
+ */
+ TriggerPolicy<String> triggerPolicy = new CountTriggerPolicy<String>(4, -1);
+ EvictionPolicy<String> evictionPolicy = new CountEvictionPolicy<String>(5, 4);
+
+ // This reduce function does a String concat.
+ ReduceFunction<String> reduceFunction = new ReduceFunction<String>() {
+
+ /**
+ * default version ID
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String reduce(String value1, String value2) throws Exception {
+ return value1 + "|" + value2;
+ }
+
+ };
+
+ DataStream<Tuple2<String, String[]>> stream = env.addSource(new CountingSource()).window(
+ triggerPolicy, evictionPolicy, reduceFunction);
+
+ stream.print();
+
+ env.execute();
+ }
+
+ @SuppressWarnings("serial")
+ private static class CountingSource implements SourceFunction<String> {
+
+ private int counter = 0;
+
+ @Override
+ public void invoke(Collector<String> collector) throws Exception {
+ // continuous emit
+ while (true) {
+ if (counter > 9999) {
+ counter = 0;
+ }
+ collector.collect("V" + counter++);
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3d11d5ba/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
new file mode 100644
index 0000000..115363b
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.util.Collector;
+
+/**
+ * This example shows the functionality of time based windows. It utilizes the
+ * {@link ActiveTriggerPolicy} implementation in the {@link TimeTriggerPolicy}.
+ */
+public class TimeWindowingExample {
+
+ private static final int PARALLELISM = 1;
+
+ public static void main(String[] args) throws Exception {
+ StreamExecutionEnvironment env = StreamExecutionEnvironment
+ .createLocalEnvironment(PARALLELISM);
+
+ // Prevent output from being blocked
+ env.setBufferTimeout(100);
+
+ // Trigger every 1000ms
+ TriggerPolicy<Integer> triggerPolicy = new TimeTriggerPolicy<Integer>(1000L,
+ new DefaultTimeStamp<Integer>(), new Extractor<Long, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer extract(Long in) {
+ return in.intValue();
+ }
+
+ });
+
+ // Always keep the newest 100 elements in the buffer
+ EvictionPolicy<Integer> evictionPolicy = new CountEvictionPolicy<Integer>(100);
+
+ // This reduce function does a String concat.
+ ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
+
+ /**
+ * default version ID
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+
+ };
+
+ DataStream<Tuple2<Integer, String[]>> stream = env.addSource(new CountingSourceWithSleep()).window(triggerPolicy, evictionPolicy, reduceFunction);
+
+ stream.print();
+
+ env.execute();
+ }
+
+ /**
+ * This data source emit one element every 0.001 sec. The output is an
+ * Integer counting the output elements. As soon as the counter reaches
+ * 10000 it is reset to 0. On each reset the source waits 5 sec. before it
+ * restarts to produce elements.
+ */
+ @SuppressWarnings("serial")
+ private static class CountingSourceWithSleep implements SourceFunction<Integer> {
+
+ private int counter = 0;
+
+ @Override
+ public void invoke(Collector<Integer> collector) throws Exception {
+ // continuous emit
+ while (true) {
+ if (counter > 9999) {
+ System.out.println("Source pauses now!");
+ Thread.sleep(5000);
+ System.out.println("Source continouse with emitting now!");
+ counter = 0;
+ }
+ collector.collect(counter);
+
+ // Wait 0.001 sec. before the next emit. Otherwise the source is
+ // too fast for local tests and you might always see
+ // SUM[k=1..9999](k) as result.
+ Thread.sleep(1);
+
+ counter++;
+ }
+ }
+
+ }
+}
[12/34] incubator-flink git commit: [streaming] Adding unit tests for
the grouped windowing invokable.
Posted by mb...@apache.org.
[streaming] Adding unit tests for the grouped windowing invokable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/751f1017
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/751f1017
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/751f1017
Branch: refs/heads/master
Commit: 751f1017c5c79be71cfef7a0bdc6272ea48881dc
Parents: 70079c1
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Thu Nov 20 15:12:38 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../operator/GroupedWindowingInvokableTest.java | 428 +++++++++++++++++++
1 file changed, 428 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/751f1017/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
new file mode 100644
index 0000000..93848db
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
@@ -0,0 +1,428 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.MockInvokable;
+import org.apache.flink.streaming.util.keys.TupleKeySelector;
+import org.junit.Test;
+
+public class GroupedWindowingInvokableTest {
+
+ /**
+ * Test for not active distributed triggers with single field
+ */
+ @Test
+ public void testGroupedWindowingInvokableDistributedTriggerSimple() {
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(1);
+ inputs.add(5);
+ inputs.add(5);
+ inputs.add(5);
+ inputs.add(1);
+ inputs.add(1);
+ inputs.add(5);
+ inputs.add(1);
+ inputs.add(5);
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(15);
+ expected.add(3);
+ expected.add(3);
+ expected.add(15);
+
+ LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
+
+ GroupedWindowingInvokable<Integer> invokable = new GroupedWindowingInvokable<Integer>(
+ new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ }, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, triggers, evictions, centralTriggers);
+
+ List<Tuple2<Integer, String[]>> result = MockInvokable.createAndExecute(invokable, inputs);
+
+ List<Integer> actual = new LinkedList<Integer>();
+ for (Tuple2<Integer, String[]> current : result) {
+ actual.add(current.f0);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(actual));
+ assertEquals(expected.size(), actual.size());
+ }
+
+ /**
+ * Test for non active distributed triggers with separated key field
+ */
+ @Test
+ public void testGroupedWindowingInvokableDistributedTriggerComplex() {
+ List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
+ inputs2.add(new Tuple2<Integer, String>(1, "a"));
+ inputs2.add(new Tuple2<Integer, String>(0, "b"));
+ inputs2.add(new Tuple2<Integer, String>(2, "a"));
+ inputs2.add(new Tuple2<Integer, String>(-1, "a"));
+ inputs2.add(new Tuple2<Integer, String>(-2, "a"));
+ inputs2.add(new Tuple2<Integer, String>(10, "a"));
+ inputs2.add(new Tuple2<Integer, String>(2, "b"));
+ inputs2.add(new Tuple2<Integer, String>(1, "a"));
+
+ List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
+ expected2.add(new Tuple2<Integer, String>(-1, "a"));
+ expected2.add(new Tuple2<Integer, String>(-2, "a"));
+ expected2.add(new Tuple2<Integer, String>(0, "b"));
+
+ LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
+
+ LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
+
+ LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+
+ GroupedWindowingInvokable<Tuple2<Integer, String>> invokable2 = new GroupedWindowingInvokable<Tuple2<Integer, String>>(
+ new ReduceFunction<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+ Tuple2<Integer, String> value2) throws Exception {
+ if (value1.f0 <= value2.f0) {
+ return value1;
+ } else {
+ return value2;
+ }
+ }
+ }, new TupleKeySelector<Tuple2<Integer, String>>(1), triggers, evictions,
+ centralTriggers);
+
+ List<Tuple2<Tuple2<Integer, String>, String[]>> result = MockInvokable.createAndExecute(
+ invokable2, inputs2);
+
+ List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
+ for (Tuple2<Tuple2<Integer, String>, String[]> current : result) {
+ actual2.add(current.f0);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
+ new HashSet<Tuple2<Integer, String>>(actual2));
+ assertEquals(expected2.size(), actual2.size());
+ }
+
+ /**
+ * Test for active centralized trigger
+ */
+ @Test
+ public void testGroupedWindowingInvokableCentralActiveTrigger() {
+
+ List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
+ inputs.add(new Tuple2<Integer, String>(1, "a"));
+ inputs.add(new Tuple2<Integer, String>(1, "b"));
+ inputs.add(new Tuple2<Integer, String>(1, "c"));
+ inputs.add(new Tuple2<Integer, String>(2, "a"));
+ inputs.add(new Tuple2<Integer, String>(2, "b"));
+ inputs.add(new Tuple2<Integer, String>(2, "c"));
+ inputs.add(new Tuple2<Integer, String>(2, "b"));
+ inputs.add(new Tuple2<Integer, String>(2, "a"));
+ inputs.add(new Tuple2<Integer, String>(2, "c"));
+ inputs.add(new Tuple2<Integer, String>(3, "c"));
+ inputs.add(new Tuple2<Integer, String>(3, "a"));
+ inputs.add(new Tuple2<Integer, String>(3, "b"));
+ inputs.add(new Tuple2<Integer, String>(4, "a"));
+ inputs.add(new Tuple2<Integer, String>(4, "b"));
+ inputs.add(new Tuple2<Integer, String>(4, "c"));
+ inputs.add(new Tuple2<Integer, String>(5, "c"));
+ inputs.add(new Tuple2<Integer, String>(5, "a"));
+ inputs.add(new Tuple2<Integer, String>(5, "b"));
+ inputs.add(new Tuple2<Integer, String>(10, "b"));
+ inputs.add(new Tuple2<Integer, String>(10, "a"));
+ inputs.add(new Tuple2<Integer, String>(10, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "a"));
+ inputs.add(new Tuple2<Integer, String>(11, "a"));
+ inputs.add(new Tuple2<Integer, String>(11, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "c"));
+ inputs.add(new Tuple2<Integer, String>(11, "b"));
+ inputs.add(new Tuple2<Integer, String>(11, "b"));
+
+ // Expected result:
+ // For each group (a,b and c):
+ // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+ // 12-12-5-10-32
+
+ List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
+ expected.add(new Tuple2<Integer, String>(12, "a"));
+ expected.add(new Tuple2<Integer, String>(12, "b"));
+ expected.add(new Tuple2<Integer, String>(12, "c"));
+ expected.add(new Tuple2<Integer, String>(12, "a"));
+ expected.add(new Tuple2<Integer, String>(12, "b"));
+ expected.add(new Tuple2<Integer, String>(12, "c"));
+ expected.add(new Tuple2<Integer, String>(5, "a"));
+ expected.add(new Tuple2<Integer, String>(5, "b"));
+ expected.add(new Tuple2<Integer, String>(5, "c"));
+ expected.add(new Tuple2<Integer, String>(10, "a"));
+ expected.add(new Tuple2<Integer, String>(10, "b"));
+ expected.add(new Tuple2<Integer, String>(10, "c"));
+ expected.add(new Tuple2<Integer, String>(32, "a"));
+ expected.add(new Tuple2<Integer, String>(32, "b"));
+ expected.add(new Tuple2<Integer, String>(32, "c"));
+
+ TimeStamp<Tuple2<Integer, String>> myTimeStamp = new TimeStamp<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Tuple2<Integer, String> value) {
+ return value.f0;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 1;
+ }
+ };
+
+ ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+ Tuple2<Integer, String> value2) throws Exception {
+ return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
+ }
+ };
+
+ LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+ // Trigger every 2 time units but delay the first trigger by 2 (First
+ // trigger after 4, then every 2)
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L,
+ new Extractor<Long, Tuple2<Integer, String>>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Integer, String> extract(Long in) {
+ return new Tuple2<Integer, String>(in.intValue(), null);
+ }
+ }));
+
+ LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+ // Always delete all elements older then 4
+ evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStamp));
+
+ LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+
+ GroupedWindowingInvokable<Tuple2<Integer, String>> invokable = new GroupedWindowingInvokable<Tuple2<Integer, String>>(
+ myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
+ distributedTriggers, evictions, triggers);
+
+ ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
+ for (Tuple2<Tuple2<Integer, String>, String[]> t : MockInvokable.createAndExecute(
+ invokable, inputs)) {
+ result.add(t.f0);
+ }
+
+ assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
+ new HashSet<Tuple2<Integer, String>>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+ /**
+ * Test for multiple centralized trigger
+ */
+ @Test
+ public void testGroupedWindowingInvokableMultipleCentralTrigger() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(8));
+ triggers.add(new CountTriggerPolicy<Integer>(5));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // The active wrapper causes eviction even on (fake) elements which
+ // triggered, but does not belong to the group.
+ evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+ new TumblingEvictionPolicy<Integer>()));
+
+ LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(1);
+ // 1st Trigger: 2;6
+ inputs.add(2);
+ inputs.add(1);
+ inputs.add(2);
+ // 2nd Trigger: 1;4
+ inputs.add(2);
+ inputs.add(1);
+ // Final: 1,2
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(2);
+ expected.add(6);
+ expected.add(4);
+ expected.add(1);
+ expected.add(2);
+ expected.add(1);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ GroupedWindowingInvokable<Integer> invokable = new GroupedWindowingInvokable<Integer>(
+ myReduceFunction, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, distributedTriggers, evictions, triggers);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t.f0);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+ /**
+ * Test for combination of centralized trigger and
+ * distributed trigger at the same time
+ */
+ @Test
+ public void testGroupedWindowingInvokableCentralAndDistrTrigger() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(8));
+ triggers.add(new CountTriggerPolicy<Integer>(5));
+
+ LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+ // The active wrapper causes eviction even on (fake) elements which
+ // triggered, but does not belong to the group.
+ evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+ new TumblingEvictionPolicy<Integer>()));
+
+ LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+ distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ // local on 2 => 4
+ inputs.add(2);
+ inputs.add(1);
+ // local on 1 => 2
+ // and 1st Central: 2;2
+ // SUMS up to 2;2
+ inputs.add(2);
+ // local on 2 => 2
+ inputs.add(1);
+ inputs.add(2);
+ // 2nd Central: 1;2
+ inputs.add(2);
+ inputs.add(1);
+ // Final: 1,2
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(4);
+ expected.add(2);
+ expected.add(2);
+ expected.add(2);
+ expected.add(1);
+ expected.add(2);
+ expected.add(1);
+ expected.add(2);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ GroupedWindowingInvokable<Integer> invokable = new GroupedWindowingInvokable<Integer>(
+ myReduceFunction, new KeySelector<Integer, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer getKey(Integer value) {
+ return value;
+ }
+ }, distributedTriggers, evictions, triggers);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t.f0);
+ }
+
+ assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+ assertEquals(expected.size(), result.size());
+ }
+
+}
[16/34] incubator-flink git commit: [streaming] Integrated the policy
based windowing into DataStream and introduced WindowedDataStream to handle
windowing helper.
Posted by mb...@apache.org.
[streaming] Integrated the policy based windowing into DataStream and introduced WindowedDataStream to handle windowing helper.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/542c66f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/542c66f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/542c66f7
Branch: refs/heads/master
Commit: 542c66f7d31e05154140299a26751b8fcf0cee0c
Parents: 4d586e7
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Tue Oct 28 10:08:31 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../streaming/api/datastream/DataStream.java | 20 +++
.../api/datastream/WindowedDataStream.java | 163 +++++++++++++++++++
2 files changed, 183 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/542c66f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 6981af4..c38a638 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -60,6 +60,10 @@ import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable;
import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.api.windowing.helper.Delta;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
import org.apache.flink.streaming.partitioner.BroadcastPartitioner;
import org.apache.flink.streaming.partitioner.DistributePartitioner;
import org.apache.flink.streaming.partitioner.FieldsPartitioner;
@@ -659,6 +663,22 @@ public class DataStream<OUT> {
return new GroupedDataStream<OUT>(this, keySelector);
}
+
+ /**
+ * This allows you to set up windowing through a nice API using
+ * {@link WindowingHelper} such as {@link Time}, {@link Count} and
+ * {@link Delta}.
+ *
+ * @param policyHelpers
+ * Any {@link WindowingHelper} such as {@link Time},
+ * {@link Count} and {@link Delta}.
+ * @return A {@link WindowedDataStream} providing further operations.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public WindowedDataStream<OUT> window(WindowingHelper... policyHelpers) {
+ return new WindowedDataStream<OUT>(this, policyHelpers);
+ }
+
/**
* Collects the data stream elements into sliding batches creating a new
* {@link BatchedDataStream}. The user can apply transformations like
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/542c66f7/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
new file mode 100644
index 0000000..2e6b885
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import java.util.LinkedList;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.invokable.operator.WindowingInvokable;
+import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.serialization.CombineTypeWrapper;
+import org.apache.flink.streaming.util.serialization.FunctionTypeWrapper;
+import org.apache.flink.streaming.util.serialization.ObjectTypeWrapper;
+
+/**
+ * A {@link WindowedDataStream} represents a data stream whose elements
+ * are batched together in a sliding batch. operations like
+ * {@link #reduce(ReduceFunction)} or {@link #reduceGroup(GroupReduceFunction)}
+ * are applied for each batch and the batch is slid afterwards.
+ *
+ * @param <OUT>
+ * The output type of the {@link WindowedDataStream}
+ */
+public class WindowedDataStream<OUT> {
+
+ protected DataStream<OUT> dataStream;
+ protected boolean isGrouped;
+ protected KeySelector<OUT, ?> keySelector;
+
+ protected WindowingHelper<OUT>[] triggerPolicies;
+ protected WindowingHelper<OUT>[] evictionPolicies;
+
+ protected WindowedDataStream(DataStream<OUT> dataStream,
+ WindowingHelper<OUT>... policyHelpers) {
+ if (dataStream instanceof GroupedDataStream) {
+ this.isGrouped = true;
+ this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+ } else {
+ this.isGrouped = false;
+ }
+ this.dataStream = dataStream.copy();
+ this.triggerPolicies = policyHelpers;
+ }
+
+ protected LinkedList<TriggerPolicy<OUT>> getTriggers() {
+ LinkedList<TriggerPolicy<OUT>> triggerPolicyList = new LinkedList<TriggerPolicy<OUT>>();
+
+ for (WindowingHelper<OUT> helper : triggerPolicies) {
+ triggerPolicyList.add(helper.toTrigger());
+ }
+
+ return triggerPolicyList;
+ }
+
+ protected LinkedList<EvictionPolicy<OUT>> getEvicters() {
+ LinkedList<EvictionPolicy<OUT>> evictionPolicyList = new LinkedList<EvictionPolicy<OUT>>();
+
+ if (evictionPolicies != null) {
+ for (WindowingHelper<OUT> helper : evictionPolicies) {
+ evictionPolicyList.add(helper.toEvict());
+ }
+ } else {
+ evictionPolicyList.add(new TumblingEvictionPolicy<OUT>());
+ }
+
+ return evictionPolicyList;
+ }
+
+ protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
+ this.dataStream = windowedDataStream.dataStream.copy();
+ this.isGrouped = windowedDataStream.isGrouped;
+ this.keySelector = windowedDataStream.keySelector;
+ this.triggerPolicies = windowedDataStream.triggerPolicies;
+ this.evictionPolicies = windowedDataStream.evictionPolicies;
+
+ }
+
+ /**
+ * Groups the elements of the {@link WindowedDataStream} by the given
+ * key position to be used with grouped operators.
+ *
+ * @param keySelector
+ * The specification of the key on which the
+ * {@link WindowedDataStream} will be grouped.
+ * @return The transformed {@link WindowedDataStream}
+ */
+ public WindowedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
+ WindowedDataStream<OUT> ret = this.copy();
+ ret.dataStream = ret.dataStream.groupBy(keySelector);
+ ret.isGrouped = true;
+ ret.keySelector = keySelector;
+ return ret;
+ }
+
+ /**
+ * This is a prototype implementation for new windowing features based on
+ * trigger and eviction policies
+ *
+ * @param triggerPolicies
+ * A list of trigger policies
+ * @param evictionPolicies
+ * A list of eviction policies
+ * @param sample
+ * A sample of the OUT data type required to gather type
+ * information
+ * @return The single output operator
+ */
+ public SingleOutputStreamOperator<Tuple2<OUT, String[]>, ?> reduce(
+ ReduceFunction<OUT> reduceFunction) {
+ return dataStream.addFunction("NextGenWindowReduce", reduceFunction,
+ new FunctionTypeWrapper<OUT>(reduceFunction, ReduceFunction.class, 0),
+ new CombineTypeWrapper<OUT, String[]>(dataStream.outTypeWrapper,
+ new ObjectTypeWrapper<String[]>(new String[] { "" })),
+ new WindowingInvokable<OUT>(reduceFunction, getTriggers(), getEvicters()));
+ }
+
+ /**
+ * Gets the output type.
+ *
+ * @return The output type.
+ */
+ public TypeInformation<OUT> getOutputType() {
+ return dataStream.getOutputType();
+ }
+
+ protected WindowedDataStream<OUT> copy() {
+ return new WindowedDataStream<OUT>(this);
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ public WindowedDataStream<OUT> every(WindowingHelper... policyHelpers) {
+ WindowedDataStream<OUT> ret = this.copy();
+ if (ret.evictionPolicies == null) {
+ ret.evictionPolicies = ret.triggerPolicies;
+ ret.triggerPolicies = policyHelpers;
+ } else {
+ ret.triggerPolicies = (WindowingHelper<OUT>[]) ArrayUtils.addAll(triggerPolicies,
+ policyHelpers);
+ }
+ return ret;
+ }
+}
[09/34] incubator-flink git commit: [streaming] Created active
trigger and eviction policies based on time. Included respective policy
helper and test cases.
Posted by mb...@apache.org.
[streaming] Created active trigger and eviction policies based on time. Included respective policy helper and test cases.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/6f3bf576
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/6f3bf576
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/6f3bf576
Branch: refs/heads/master
Commit: 6f3bf576c68e1884393016ef46297236367522eb
Parents: 40c3ae4
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:31:00 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../streaming/api/windowing/helper/Time.java | 125 ++++++++++++
.../windowing/policy/TimeEvictionPolicy.java | 109 +++++++++++
.../api/windowing/policy/TimeTriggerPolicy.java | 193 +++++++++++++++++++
.../policy/TimeEvictionPolicyTest.java | 125 ++++++++++++
.../windowing/policy/TimeTriggerPolicyTest.java | 137 +++++++++++++
5 files changed, 689 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6f3bf576/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
new file mode 100644
index 0000000..37f2902
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * This helper represents a time based count or eviction policy. By default the
+ * time is measured with {@link System#currentTimeMillis()} in
+ * {@link DefaultTimeStamp}.
+ *
+ * @param <DATA>
+ * The data type which is handled by the time stamp used in the
+ * policy represented by this helper
+ */
+public class Time<DATA> implements WindowingHelper<DATA> {
+
+ private int timeVal;
+ private TimeUnit granularity;
+ private Extractor<Long, DATA> timeToData;
+
+ /**
+ * Creates an helper representing a trigger which triggers every given
+ * timeVal or an eviction which evicts all elements older than timeVal.
+ *
+ * @param timeVal
+ * The number of time units
+ * @param granularity
+ * The unit of time such as minute oder millisecond. Note that
+ * the smallest possible granularity is milliseconds. Any smaller
+ * time unit might cause an error at runtime due to conversion
+ * problems.
+ * @param timeToData
+ * This policy creates fake elements to not miss windows in case
+ * no element arrived within the duration of the window. This
+ * extractor should wrap a long into such an element of type
+ * DATA.
+ */
+ public Time(int timeVal, TimeUnit granularity, Extractor<Long, DATA> timeToData) {
+ this.timeVal = timeVal;
+ this.granularity = granularity;
+ this.timeToData = timeToData;
+ }
+
+ /**
+ * Creates an helper representing a trigger which triggers every given
+ * timeVal or an eviction which evicts all elements older than timeVal.
+ *
+ * The default granularity for timeVal used in this method is seconds.
+ *
+ * @param timeVal
+ * The number of time units measured in seconds.
+ * @param timeToData
+ * This policy creates fake elements to not miss windows in case
+ * no element arrived within the duration of the window. This
+ * extractor should wrap a long into such an element of type
+ * DATA.
+ */
+ public Time(int timeVal, Extractor<Long, DATA> timeToData) {
+ this(timeVal, TimeUnit.SECONDS, timeToData);
+ }
+
+ @Override
+ public EvictionPolicy<DATA> toEvict() {
+ return new TimeEvictionPolicy<DATA>(granularityInMillis(), new DefaultTimeStamp<DATA>());
+ }
+
+ @Override
+ public TriggerPolicy<DATA> toTrigger() {
+ return new TimeTriggerPolicy<DATA>(granularityInMillis(), new DefaultTimeStamp<DATA>(),
+ timeToData);
+ }
+
+ /**
+ * Creates an helper representing a trigger which triggers every given
+ * timeVal or an eviction which evicts all elements older than timeVal.
+ *
+ * @param timeVal
+ * The number of time units
+ * @param granularity
+ * The unit of time such as minute oder millisecond. Note that
+ * the smallest possible granularity is milliseconds. Any smaller
+ * time unit might cause an error at runtime due to conversion
+ * problems.
+ * @param timeToData
+ * This policy creates fake elements to not miss windows in case
+ * no element arrived within the duration of the window. This
+ * extractor should wrap a long into such an element of type
+ * DATA.
+ * @return an helper representing a trigger which triggers every given
+ * timeVal or an eviction which evicts all elements older than
+ * timeVal.
+ */
+ public static <DATA> Time<DATA> of(int timeVal, TimeUnit granularity,
+ Extractor<Long, DATA> timeToData) {
+ return new Time<DATA>(timeVal, granularity, timeToData);
+ }
+
+ private long granularityInMillis() {
+ return this.granularity.toMillis(this.timeVal);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6f3bf576/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
new file mode 100644
index 0000000..a78cc4e
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+
+/**
+ * This eviction policy evicts all elements which are older then a specified
+ * time. The time is measured using a given {@link TimeStamp} implementation. A
+ * point in time is always represented as long. Therefore, the granularity can
+ * be set as long value as well.
+ *
+ * @param <DATA>
+ * The type of the incoming data points which are processed by this
+ * policy.
+ */
+public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -1457476766124518220L;
+
+ private long granularity;
+ private TimeStamp<DATA> timestamp;
+ private LinkedList<DATA> buffer = new LinkedList<DATA>();
+
+ /**
+ * This eviction policy evicts all elements which are older than a specified
+ * time. The time is measured using a given {@link TimeStamp}
+ * implementation. A point in time is always represented as long. Therefore,
+ * the granularity can be set as long value as well. If this value is set to
+ * 2 the policy will evict all elements which are older as 2.
+ *
+ * <code>
+ * while (time(firstInBuffer)<current time-granularity){
+ * evict firstInBuffer;
+ * }
+ * </code>
+ *
+ * @param granularity
+ * The granularity of the eviction. If this value is set to 2 the
+ * policy will evict all elements which are older as 2(if
+ * (time(X)<current time-granularity) evict X).
+ * @param timestamp
+ * The {@link TimeStamp} to measure the time with. This can be
+ * either user defined of provided by the API.
+ */
+ public TimeEvictionPolicy(long granularity, TimeStamp<DATA> timestamp) {
+ this.timestamp = timestamp;
+ this.granularity = granularity;
+ }
+
+ @Override
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
+ return notifyEviction(datapoint, triggered, bufferSize, false);
+ }
+
+ @Override
+ public int notifyEvictionWithFakeElement(DATA datapoint, int bufferSize) {
+ return notifyEviction(datapoint, true, bufferSize, true);
+ }
+
+ private int notifyEviction(DATA datapoint, boolean triggered, int bufferSize, boolean isFake) {
+ // check for deleted tuples (deletes by other policies)
+ while (bufferSize < this.buffer.size()) {
+ this.buffer.removeFirst();
+ }
+
+ // delete and count expired tuples
+ int counter = 0;
+ while (!buffer.isEmpty()) {
+
+ if (timestamp.getTimestamp(buffer.getFirst()) < timestamp.getTimestamp(datapoint)
+ - granularity) {
+ buffer.removeFirst();
+ counter++;
+ } else {
+ break;
+ }
+ }
+
+ if (!isFake) {
+ // Add current element to buffer
+ buffer.add(datapoint);
+ }
+
+ // return result
+ return counter;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6f3bf576/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
new file mode 100644
index 0000000..d50e285
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+
+import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * This trigger policy triggers with regard to the time. The is measured using a
+ * given {@link TimeStamp} implementation. A point in time is always represented
+ * as long. Therefore, parameters such as granularity and delay can be set as
+ * long value as well.
+ *
+ * @param <DATA>
+ * The type of the incoming data points which are processed by this
+ * policy.
+ */
+public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -5122753802440196719L;
+
+ private long startTime;
+ private long granularity;
+ private TimeStamp<DATA> timestamp;
+ private Extractor<Long, DATA> longToDATAExtractor;
+
+ /**
+ * This trigger policy triggers with regard to the time. The is measured
+ * using a given {@link TimeStamp} implementation. A point in time is always
+ * represented as long. Therefore, parameters such as granularity can be set
+ * as long value as well. If this value for the granularity is set to 2 for
+ * example, the policy will trigger at every second point in time.
+ *
+ * @param granularity
+ * The granularity of the trigger. If this value is set to 2 the
+ * policy will trigger at every second time point
+ * @param timestamp
+ * The {@link TimeStamp} to measure the time with. This can be
+ * either user defined of provided by the API.
+ * @param timeWrapper
+ * This policy creates fake elements to not miss windows in case
+ * no element arrived within the duration of the window. This
+ * extractor should wrap a long into such an element of type
+ * DATA.
+ */
+ public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp,
+ Extractor<Long, DATA> timeWrapper) {
+ this(granularity, timestamp, 0, timeWrapper);
+ }
+
+ /**
+ * This is mostly the same as
+ * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, TimeStamp)}. In addition
+ * to granularity and timestamp a delay can be specified for the first
+ * trigger. If the start time given by the timestamp is x, the delay is y,
+ * and the granularity is z, the first trigger will happen at x+y+z.
+ *
+ * @param granularity
+ * The granularity of the trigger. If this value is set to 2 the
+ * policy will trigger at every second time point
+ * @param timestamp
+ * The {@link TimeStamp} to measure the time with. This can be
+ * either user defined of provided by the API.
+ * @param delay
+ * A delay for the first trigger. If the start time given by the
+ * timestamp is x, the delay is y, and the granularity is z, the
+ * first trigger will happen at x+y+z.
+ * @param timeWrapper
+ * This policy creates fake elements to not miss windows in case
+ * no element arrived within the duration of the window. This
+ * extractor should wrap a long into such an element of type
+ * DATA.
+ */
+ public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp, long delay,
+ Extractor<Long, DATA> timeWrapper) {
+ this.startTime = timestamp.getStartTime() + delay;
+ this.timestamp = timestamp;
+ this.granularity = granularity;
+ this.longToDATAExtractor = timeWrapper;
+ }
+
+ @Override
+ public synchronized boolean notifyTrigger(DATA datapoint) {
+ long recordTime = timestamp.getTimestamp(datapoint);
+ // start time is included, but end time is excluded: >=
+ if (recordTime >= startTime + granularity) {
+ if (granularity != 0) {
+ startTime = recordTime - ((recordTime - startTime) % granularity);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * This method checks if we missed a window end. If this is the
+ * case we trigger the missed windows using fake elements.
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public synchronized DATA[] preNotifyTrigger(DATA datapoint) {
+ LinkedList<DATA> fakeElements = new LinkedList<DATA>();
+ // check if there is more then one window border missed
+ // use > here. In case >= would fit, the regular call will do the job.
+ while (timestamp.getTimestamp(datapoint) > startTime + granularity) {
+ fakeElements.add(longToDATAExtractor.extract(startTime += granularity));
+ }
+ return (DATA[]) fakeElements.toArray();
+ }
+
+ /**
+ * In case {@link DefaultTimeStamp} is used, a runnable is returned which
+ * triggers based on the current system time. If any other time measure is
+ * used the method return null.
+ *
+ * @param callback
+ * The object which is takes the callbacks for adding fake
+ * elements out of the runnable.
+ * @return A runnable is returned which triggers based on the current system
+ * time. If any other time measure is used the method return null.
+ */
+ @Override
+ public Runnable createActiveTriggerRunnable(ActiveTriggerCallback<DATA> callback) {
+ if (this.timestamp instanceof DefaultTimeStamp) {
+ return new TimeCheck(callback);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * This method is only called in case the runnable triggers a window end
+ * according to the {@link DefaultTimeStamp}.
+ *
+ * @param callback
+ * The callback object.
+ */
+ private synchronized void activeFakeElementEmission(ActiveTriggerCallback<DATA> callback) {
+
+ if (System.currentTimeMillis() >= startTime + granularity) {
+ startTime += granularity;
+ callback.sendFakeElement(longToDATAExtractor.extract(startTime += granularity));
+ }
+
+ }
+
+ private class TimeCheck implements Runnable {
+ ActiveTriggerCallback<DATA> callback;
+
+ public TimeCheck(ActiveTriggerCallback<DATA> callback) {
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ // wait for the specified granularity
+ try {
+ Thread.sleep(granularity);
+ } catch (InterruptedException e) {
+ // ignore it...
+ }
+ // Trigger using the respective methods. Methods are
+ // synchronized to prevent race conditions between real and fake
+ // elements at the policy.
+ activeFakeElementEmission(callback);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6f3bf576/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
new file mode 100644
index 0000000..303c66a
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TimeEvictionPolicyTest {
+
+ @Test
+ public void timeEvictionTest() {
+ // create some test data
+ Integer[] times = { 1, 3, 4, 6, 7, 9, 14, 20, 21, 22, 30, 31, 33, 36, 40, 41, 42, 43, 44,
+ 45, 47, 55 };
+ Integer[] numToDelete = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 3 };
+
+ // create a timestamp
+ @SuppressWarnings("serial")
+ TimeStamp<Integer> timeStamp = new TimeStamp<Integer>() {
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 0;
+ }
+
+ };
+
+ // test different granularity
+ for (long granularity = 0; granularity < 40; granularity++) {
+ // create policy
+ TimeEvictionPolicy<Integer> policy = new TimeEvictionPolicy<Integer>(granularity,
+ timeStamp);
+
+ // The trigger status should not effect the policy. Therefore, it's
+ // value is changed after each usage.
+ boolean triggered = false;
+
+ // The eviction should work similar with both, fake and real
+ // elements. Which kind is used is changed on every 3rd element in
+ // this test.
+ int fakeAndRealCounter = 0;
+ boolean fake = false;
+
+ // test by adding values
+ LinkedList<Integer> buffer = new LinkedList<Integer>();
+ for (int i = 0; i < times.length; i++) {
+
+ // check if the current element should be a fake
+ fakeAndRealCounter++;
+ if (fakeAndRealCounter > 2) {
+ fake = !fake;
+ fakeAndRealCounter = 0;
+ }
+
+ int result;
+
+ if (fake) {
+ // Notify eviction with fake element
+ result = policy.notifyEvictionWithFakeElement(times[i], buffer.size());
+ } else {
+ // Notify eviction with real element
+ result = policy.notifyEviction(times[i], (triggered = !triggered),
+ buffer.size());
+ }
+
+ // handle correctness of eviction
+ for (; result > 0 && !buffer.isEmpty(); result--) {
+ if (buffer.getFirst() < times[i] - granularity) {
+ buffer.removeFirst();
+ } else {
+ fail("The policy wanted to evict time " + buffer.getFirst()
+ + " while the current time was " + times[i]
+ + "and the granularity was " + granularity);
+ }
+ }
+
+ // test that all required evictions have been done
+ if (!buffer.isEmpty()) {
+ assertTrue("The policy did not evict " + buffer.getFirst()
+ + " while the current time was " + times[i]
+ + " and the granularity was " + granularity,
+ (buffer.getFirst() >= times[i] - granularity));
+ }
+
+ // test influence of other evictions
+ for (int j = numToDelete[i % numToDelete.length]; j > 0; j--) {
+ if (!buffer.isEmpty()) {
+ buffer.removeFirst();
+ }
+ }
+
+ // add current element to buffer if it is no fake
+ if (!fake) {
+ buffer.add(times[i]);
+ }
+
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/6f3bf576/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
new file mode 100644
index 0000000..839564d
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TimeTriggerPolicyTest {
+
+ @Test
+ public void timeTriggerRegularNotifyTest() {
+ // create some test data
+ Integer[] times = { 1, 3, 4, 6, 7, 9, 14, 20, 21, 22, 30 };
+
+ // create a timestamp
+ @SuppressWarnings("serial")
+ TimeStamp<Integer> timeStamp = new TimeStamp<Integer>() {
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 0;
+ }
+
+ };
+
+ // test different granularity
+ for (long granularity = 0; granularity < 31; granularity++) {
+ // create policy
+ TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity, timeStamp,
+ new Extractor<Long, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer extract(Long in) {
+ return in.intValue();
+ }
+ });
+
+ // remember window border
+ // Remark: This might NOT work in case the timeStamp uses
+ // System.getCurrentTimeMillis to determine the start time.
+ long currentTime = timeStamp.getStartTime();
+
+ // test by adding values
+ for (int i = 0; i < times.length; i++) {
+ boolean result = policy.notifyTrigger(times[i]);
+ // start time is included, but end time is excluded: >=
+ if (times[i] >= currentTime + granularity) {
+ if (granularity != 0) {
+ currentTime = times[i] - ((times[i] - currentTime) % granularity);
+ }
+ assertTrue("The policy did not trigger at pos " + i + " (current time border: "
+ + currentTime + "; current granularity: " + granularity
+ + "; data point time: " + times[i] + ")", result);
+ } else {
+ assertFalse("The policy triggered wrong at pos " + i
+ + " (current time border: " + currentTime + "; current granularity: "
+ + granularity + "; data point time: " + times[i] + ")", result);
+ }
+ }
+ }
+
+ }
+
+ @Test
+ public void timeTriggerPreNotifyTest() {
+ // create some test data
+ Integer[] times = { 1, 3, 20, 26};
+
+ // create a timestamp
+ @SuppressWarnings("serial")
+ TimeStamp<Integer> timeStamp = new TimeStamp<Integer>() {
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 0;
+ }
+
+ };
+
+ // create policy
+ TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5, timeStamp,
+ new Extractor<Long, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer extract(Long in) {
+ return in.intValue();
+ }
+ });
+
+ //expected result
+ Integer[][] result={{},{},{5,10,15},{25}};
+
+ //call policy
+ for (int i=0;i<times.length;i++){
+ arrayEqualityCheck(result[i],policy.preNotifyTrigger(times[i]));
+ policy.notifyTrigger(times[i]);
+ }
+ }
+
+ private void arrayEqualityCheck(Object[] array1, Object[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
+ }
+ }
+
+}
[21/34] incubator-flink git commit: [streaming] Window cross API
rework
Posted by mb...@apache.org.
[streaming] Window cross API rework
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/64baa00b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/64baa00b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/64baa00b
Branch: refs/heads/master
Commit: 64baa00b130bae4d602ccafd52f5a687d6042279
Parents: d6dc434
Author: ghermann <re...@gmail.com>
Authored: Wed Nov 26 19:55:03 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../api/datastream/ConnectedDataStream.java | 79 +++++++++----
.../streaming/api/datastream/DataStream.java | 76 +++++--------
.../api/datastream/StreamCrossOperator.java | 69 +++++++++++
.../api/datastream/StreamJoinOperator.java | 114 +++++--------------
.../api/datastream/WindowDBOperator.java | 103 +++++++++++++++++
.../streaming/api/WindowCrossJoinTest.java | 19 +++-
6 files changed, 301 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/64baa00b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
index d2b8332..1621752 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/ConnectedDataStream.java
@@ -18,9 +18,11 @@
package org.apache.flink.streaming.api.datastream;
import java.io.Serializable;
+import java.util.List;
import org.apache.commons.lang3.SerializationException;
import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -32,7 +34,6 @@ import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
import org.apache.flink.streaming.api.function.co.CoMapFunction;
import org.apache.flink.streaming.api.function.co.CoReduceFunction;
import org.apache.flink.streaming.api.function.co.CoWindowFunction;
-import org.apache.flink.streaming.api.function.co.CrossWindowFunction;
import org.apache.flink.streaming.api.function.co.RichCoMapFunction;
import org.apache.flink.streaming.api.function.co.RichCoReduceFunction;
import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
@@ -47,6 +48,7 @@ import org.apache.flink.streaming.util.serialization.CombineTypeWrapper;
import org.apache.flink.streaming.util.serialization.FunctionTypeWrapper;
import org.apache.flink.streaming.util.serialization.ObjectTypeWrapper;
import org.apache.flink.streaming.util.serialization.TypeWrapper;
+import org.apache.flink.util.Collector;
/**
* The ConnectedDataStream represents a stream for two different data types. It
@@ -550,22 +552,66 @@ public class ConnectedDataStream<IN1, IN2> {
return invokable;
}
- SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> windowCross(long windowSize, long slideInterval) {
- return windowCross(windowSize, slideInterval, new DefaultTimeStamp<IN1>(),
- new DefaultTimeStamp<IN2>());
- }
+ protected <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCross(
+ CrossFunction<IN1, IN2, OUT> crossFunction, long windowSize, long slideInterval,
+ TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
- SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> windowCross(long windowSize,
- long slideInterval, TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
+ TypeWrapper<IN1> in1TypeWrapper = new ObjectTypeWrapper<IN1>(dataStream1.getOutputType()
+ .createSerializer().createInstance());
+ TypeWrapper<IN2> in2TypeWrapper = new ObjectTypeWrapper<IN2>(dataStream2.getOutputType()
+ .createSerializer().createInstance());
+
+ FunctionTypeWrapper<OUT> outTypeWrapper = new FunctionTypeWrapper<OUT>(crossFunction,
+ CrossFunction.class, 2);
- return addGeneralWindowJoin(new CrossWindowFunction<IN1, IN2>(), windowSize, slideInterval,
- timestamp1, timestamp2);
+ CrossWindowFunction<IN1, IN2, OUT> crossWindowFunction = new CrossWindowFunction<IN1, IN2, OUT>(crossFunction);
+
+ return addGeneralWindowCombine(crossWindowFunction, in1TypeWrapper, in2TypeWrapper,
+ outTypeWrapper, windowSize, slideInterval, timestamp1, timestamp2);
}
+ private static class CrossWindowFunction<IN1, IN2, OUT> implements CoWindowFunction<IN1, IN2, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ private CrossFunction<IN1, IN2, OUT> crossFunction;
+
+ public CrossWindowFunction(CrossFunction<IN1, IN2, OUT> crossFunction) {
+ this.crossFunction = crossFunction;
+ }
+
+ @Override
+ public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out)
+ throws Exception {
+ for (IN1 firstValue : first) {
+ for (IN2 secondValue : second) {
+ out.collect(crossFunction.cross(firstValue, secondValue));
+ }
+ }
+ }
+ }
+
protected SingleOutputStreamOperator<Tuple2<IN1, IN2>, ?> addGeneralWindowJoin(
CoWindowFunction<IN1, IN2, Tuple2<IN1, IN2>> coWindowFunction, long windowSize,
long slideInterval, TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
+ TypeWrapper<IN1> in1TypeWrapper = new ObjectTypeWrapper<IN1>(dataStream1.getOutputType()
+ .createSerializer().createInstance());
+ TypeWrapper<IN2> in2TypeWrapper = new ObjectTypeWrapper<IN2>(dataStream2.getOutputType()
+ .createSerializer().createInstance());
+
+ CombineTypeWrapper<IN1, IN2> outTypeWrapper = new CombineTypeWrapper<IN1, IN2>(
+ in1TypeWrapper, in2TypeWrapper);
+
+ return addGeneralWindowCombine(coWindowFunction, in1TypeWrapper, in2TypeWrapper,
+ outTypeWrapper, windowSize, slideInterval, timestamp1, timestamp2);
+ }
+
+ private <OUT> SingleOutputStreamOperator<OUT, ?> addGeneralWindowCombine(
+ CoWindowFunction<IN1, IN2, OUT> coWindowFunction, TypeWrapper<IN1> in1TypeWrapper,
+ TypeWrapper<IN2> in2TypeWrapper, TypeWrapper<OUT> outTypeWrapper, long windowSize,
+ long slideInterval, TimeStamp<IN1> timestamp1, TimeStamp<IN2> timestamp2) {
+
if (windowSize < 1) {
throw new IllegalArgumentException("Window size must be positive");
}
@@ -573,20 +619,9 @@ public class ConnectedDataStream<IN1, IN2> {
throw new IllegalArgumentException("Slide interval must be positive");
}
- TypeWrapper<IN1> in1TypeWrapper = null;
- TypeWrapper<IN2> in2TypeWrapper = null;
-
- in1TypeWrapper = new ObjectTypeWrapper<IN1>(dataStream1.getOutputType().createSerializer()
- .createInstance());
- in2TypeWrapper = new ObjectTypeWrapper<IN2>(dataStream2.getOutputType().createSerializer()
- .createInstance());
-
- CombineTypeWrapper<IN1, IN2> outTypeWrapper = new CombineTypeWrapper<IN1, IN2>(
- in1TypeWrapper, in2TypeWrapper);
-
return addCoFunction("coWindowReduce", coWindowFunction, in1TypeWrapper, in2TypeWrapper,
- outTypeWrapper, new CoWindowInvokable<IN1, IN2, Tuple2<IN1, IN2>>(coWindowFunction,
- windowSize, slideInterval, timestamp1, timestamp2));
+ outTypeWrapper, new CoWindowInvokable<IN1, IN2, OUT>(coWindowFunction, windowSize,
+ slideInterval, timestamp1, timestamp2));
}
protected <OUT> SingleOutputStreamOperator<OUT, ?> addCoFunction(String functionName,
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/64baa00b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 13a7ba1..fd19f73 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -37,7 +37,6 @@ import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.streaming.api.JobGraphBuilder;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -58,8 +57,6 @@ import org.apache.flink.streaming.api.invokable.operator.FilterInvokable;
import org.apache.flink.streaming.api.invokable.operator.FlatMapInvokable;
import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
import org.apache.flink.streaming.api.invokable.operator.StreamReduceInvokable;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.windowing.helper.Count;
import org.apache.flink.streaming.api.windowing.helper.Delta;
import org.apache.flink.streaming.api.windowing.helper.Time;
@@ -496,6 +493,34 @@ public class DataStream<OUT> {
}
/**
+ * Initiates a temporal Cross transformation.<br/>
+ * A Cross transformation combines the elements of two {@link DataStream}s
+ * into one DataStream over a specified time window. It builds all pair
+ * combinations of elements of both DataStreams, i.e., it builds a Cartesian
+ * product.
+ *
+ * <p>
+ * This method returns a {@link StreamCrossOperator} on which the
+ * {@link StreamCrossOperator#onWindow} should be called to define the
+ * window, and then call
+ * {@link StreamCrossOperator.CrossWindow#with(org.apache.flink.api.common.functions.CrossFunction)}
+ * to define a {@link org.apache.flink.api.common.functions.CrossFunction}
+ * which is called for each pair of crossed elements. The CrossFunction
+ * returns a exactly one element for each pair of input elements.
+ *
+ * @param dataStreamToCross
+ * The other DataStream with which this DataStream is crossed.
+ * @return A {@link StreamCrossOperator} to continue the definition of the
+ * Join transformation.
+ *
+ * @see org.apache.flink.api.common.functions.CrossFunction
+ * @see DataStream
+ */
+ public <IN2> StreamCrossOperator<OUT, IN2> cross(DataStream<IN2> dataStreamToCross) {
+ return new StreamCrossOperator<OUT, IN2>(this, dataStreamToCross);
+ }
+
+ /**
* Initiates a temporal Join transformation. <br/>
* A temporal Join transformation joins the elements of two
* {@link DataStream}s on key equality over a specified time window.</br>
@@ -766,51 +791,6 @@ public class DataStream<OUT> {
}
/**
- * Creates a cross (Cartesian product) of a data stream window. The user can
- * implement their own time stamps or use the system time by default.
- *
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param dataStreamToCross
- * @param windowSize
- * @param slideInterval
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowCross(
- DataStream<IN2> dataStreamToCross, long windowSize, long slideInterval) {
- return this.windowCross(dataStreamToCross, windowSize, slideInterval,
- new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>());
- }
-
- /**
- * Creates a cross (Cartesian product) of a data stream window.
- *
- * @param dataStreamToCross
- * {@link DataStream} to cross with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param timestamp1
- * User defined time stamps for the first input.
- * @param timestamp2
- * User defined time stamps for the second input.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowCross(
- DataStream<IN2> dataStreamToCross, long windowSize, long slideInterval,
- TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2) {
- return this.connect(dataStreamToCross).windowCross(windowSize, slideInterval, timestamp1,
- timestamp2);
- }
-
- /**
* Writes a DataStream to the standard output stream (stdout). For each
* element of the DataStream the result of {@link Object#toString()} is
* written.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/64baa00b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
new file mode 100644
index 0000000..f54f273
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamCrossOperator.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.functions.CrossFunction;
+import org.apache.flink.api.java.DataSet;
+
+public class StreamCrossOperator<I1, I2> extends WindowDBOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
+
+ public StreamCrossOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ super(input1, input2);
+ }
+
+ @Override
+ protected CrossWindow<I1, I2> createNextWindowOperator() {
+ return new CrossWindow<I1, I2>(this);
+ }
+
+ public static class CrossWindow<I1, I2> {
+
+ private StreamCrossOperator<I1, I2> op;
+
+ public CrossWindow(StreamCrossOperator<I1, I2> operator) {
+ this.op = operator;
+ }
+
+ /**
+ * Finalizes a temporal Cross transformation by applying a {@link CrossFunction} to each pair of crossed elements.<br/>
+ * Each CrossFunction call returns exactly one element.
+ *
+ * @param function The CrossFunction that is called for each pair of crossed elements.
+ * @return An CrossOperator that represents the crossed result DataSet
+ *
+ * @see CrossFunction
+ * @see DataSet
+ */
+ public <R> SingleOutputStreamOperator<R, ?> with(CrossFunction<I1, I2, R> function) {
+ return createCrossOperator(function);
+ }
+
+ protected <R> SingleOutputStreamOperator<R, ?> createCrossOperator(
+ CrossFunction<I1, I2, R> function) {
+
+ return op.input1.connect(op.input2).addGeneralWindowCross(function, op.windowSize,
+ op.slideInterval, op.timeStamp1, op.timeStamp2);
+
+ }
+
+ // ----------------------------------------------------------------------------------------
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/64baa00b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
index fb76d28..ba6e75e 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/StreamJoinOperator.java
@@ -22,92 +22,27 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.function.co.JoinWindowFunction;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.util.keys.FieldsKeySelector;
import org.apache.flink.streaming.util.keys.PojoKeySelector;
-public class StreamJoinOperator<I1, I2> {
-
- private final DataStream<I1> input1;
- private final DataStream<I2> input2;
-
- long windowSize;
- long slideInterval;
-
- TimeStamp<I1> timeStamp1;
- TimeStamp<I2> timeStamp2;
+public class StreamJoinOperator<I1, I2> extends
+ WindowDBOperator<I1, I2, StreamJoinOperator.JoinWindow<I1, I2>> {
public StreamJoinOperator(DataStream<I1> input1, DataStream<I2> input2) {
- if (input1 == null || input2 == null) {
- throw new NullPointerException();
- }
- this.input1 = input1.copy();
- this.input2 = input2.copy();
+ super(input1, input2);
}
- /**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
- */
- public JoinWindow onWindow(long windowSize) {
- return onWindow(windowSize, windowSize);
- }
-
- /**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param slideInterval
- * The slide size of the window.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
- */
- public JoinWindow onWindow(long windowSize, long slideInterval) {
- return onWindow(windowSize, slideInterval, new DefaultTimeStamp<I1>(),
- new DefaultTimeStamp<I2>());
- }
-
- /**
- * Continues a temporal Join transformation.<br/>
- * Defines the window size on which the two DataStreams will be joined.
- *
- * @param windowSize
- * The size of the window in milliseconds.
- * @param slideInterval
- * The slide size of the window.
- * @param timeStamp1
- * The timestamp used to extract time from the elements of the
- * first data stream.
- * @param timeStamp2
- * The timestamp used to extract time from the elements of the
- * second data stream.
- * @return An incomplete Join transformation. Call {@link JoinWindow#where}
- * to continue the Join.
- */
- public JoinWindow onWindow(long windowSize, long slideInterval, TimeStamp<I1> timeStamp1,
- TimeStamp<I2> timeStamp2) {
-
- this.windowSize = windowSize;
- this.slideInterval = slideInterval;
-
- this.timeStamp1 = timeStamp1;
- this.timeStamp2 = timeStamp2;
-
- return new JoinWindow();
+ @Override
+ protected JoinWindow<I1, I2> createNextWindowOperator() {
+ return new JoinWindow<I1, I2>(this);
}
- public class JoinWindow {
+ public static class JoinWindow<I1, I2> {
- private JoinWindow() {
+ private StreamJoinOperator<I1, I2> op;
+ private JoinWindow(StreamJoinOperator<I1, I2> operator) {
+ this.op = operator;
}
/**
@@ -123,8 +58,9 @@ public class StreamJoinOperator<I1, I2> {
* @return An incomplete Join transformation. Call
* {@link JoinPredicate#equalTo} to continue the Join.
*/
- public JoinPredicate where(int... fields) {
- return new JoinPredicate(FieldsKeySelector.getSelector(input1.getOutputType(), fields));
+ public JoinPredicate<I1, I2> where(int... fields) {
+ return new JoinPredicate<I1, I2>(op, FieldsKeySelector.getSelector(
+ op.input1.getOutputType(), fields));
}
/**
@@ -139,8 +75,9 @@ public class StreamJoinOperator<I1, I2> {
* @return An incomplete Join transformation. Call
* {@link JoinPredicate#equalTo} to continue the Join.
*/
- public JoinPredicate where(String... fields) {
- return new JoinPredicate(new PojoKeySelector<I1>(input1.getOutputType(), fields));
+ public JoinPredicate<I1, I2> where(String... fields) {
+ return new JoinPredicate<I1, I2>(op, new PojoKeySelector<I1>(op.input1.getOutputType(),
+ fields));
}
/**
@@ -156,8 +93,8 @@ public class StreamJoinOperator<I1, I2> {
* @return An incomplete Join transformation. Call
* {@link JoinPredicate#equalTo} to continue the Join.
*/
- public <K> JoinPredicate where(KeySelector<I1, K> keySelector) {
- return new JoinPredicate(keySelector);
+ public <K> JoinPredicate<I1, I2> where(KeySelector<I1, K> keySelector) {
+ return new JoinPredicate<I1, I2>(op, keySelector);
}
// ----------------------------------------------------------------------------------------
@@ -170,11 +107,13 @@ public class StreamJoinOperator<I1, I2> {
* input {@link DataStream} by calling {@link JoinPredicate#equalTo}
*
*/
- public class JoinPredicate {
+ public static class JoinPredicate<I1, I2> {
+ private StreamJoinOperator<I1, I2> op;
private final KeySelector<I1, ?> keys1;
- private JoinPredicate(KeySelector<I1, ?> keys1) {
+ private JoinPredicate(StreamJoinOperator<I1, I2> operator, KeySelector<I1, ?> keys1) {
+ this.op = operator;
this.keys1 = keys1;
}
@@ -196,7 +135,8 @@ public class StreamJoinOperator<I1, I2> {
* @return The joined data stream.
*/
public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(int... fields) {
- return createJoinOperator(FieldsKeySelector.getSelector(input2.getOutputType(), fields));
+ return createJoinOperator(FieldsKeySelector.getSelector(op.input2.getOutputType(),
+ fields));
}
/**
@@ -214,7 +154,7 @@ public class StreamJoinOperator<I1, I2> {
* @return The joined data stream.
*/
public SingleOutputStreamOperator<Tuple2<I1, I2>, ?> equalTo(String... fields) {
- return createJoinOperator(new PojoKeySelector<I2>(input2.getOutputType(), fields));
+ return createJoinOperator(new PojoKeySelector<I2>(op.input2.getOutputType(), fields));
}
/**
@@ -244,8 +184,8 @@ public class StreamJoinOperator<I1, I2> {
JoinWindowFunction<I1, I2> joinWindowFunction = new JoinWindowFunction<I1, I2>(keys1,
keys2);
- return input1.connect(input2).addGeneralWindowJoin(joinWindowFunction, windowSize,
- slideInterval, timeStamp1, timeStamp2);
+ return op.input1.connect(op.input2).addGeneralWindowJoin(joinWindowFunction,
+ op.windowSize, op.slideInterval, op.timeStamp1, op.timeStamp2);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/64baa00b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java
new file mode 100644
index 0000000..4f6f0c1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDBOperator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+
+public abstract class WindowDBOperator<I1, I2, OP> {
+
+ protected final DataStream<I1> input1;
+ protected final DataStream<I2> input2;
+
+ long windowSize;
+ long slideInterval;
+
+ TimeStamp<I1> timeStamp1;
+ TimeStamp<I2> timeStamp2;
+
+ public WindowDBOperator(DataStream<I1> input1, DataStream<I2> input2) {
+ if (input1 == null || input2 == null) {
+ throw new NullPointerException();
+ }
+ this.input1 = input1.copy();
+ this.input2 = input2.copy();
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public OP onWindow(long windowSize) {
+ return onWindow(windowSize, windowSize);
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param slideInterval
+ * The slide size of the window.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public OP onWindow(long windowSize, long slideInterval) {
+ return onWindow(windowSize, slideInterval, new DefaultTimeStamp<I1>(),
+ new DefaultTimeStamp<I2>());
+ }
+
+ /**
+ * Continues a temporal Join transformation.<br/>
+ * Defines the window size on which the two DataStreams will be joined.
+ *
+ * @param windowSize
+ * The size of the window in milliseconds.
+ * @param slideInterval
+ * The slide size of the window.
+ * @param timeStamp1
+ * The timestamp used to extract time from the elements of the
+ * first data stream.
+ * @param timeStamp2
+ * The timestamp used to extract time from the elements of the
+ * second data stream.
+ * @return An incomplete Join transformation. Call {@link JoinWindow#where}
+ * to continue the Join.
+ */
+ public OP onWindow(long windowSize, long slideInterval, TimeStamp<I1> timeStamp1,
+ TimeStamp<I2> timeStamp2) {
+
+ this.windowSize = windowSize;
+ this.slideInterval = slideInterval;
+
+ this.timeStamp1 = timeStamp1;
+ this.timeStamp2 = timeStamp2;
+
+ return createNextWindowOperator();
+ }
+
+ protected abstract OP createNextWindowOperator();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/64baa00b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
index 3b719be..d8cdfa5 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/WindowCrossJoinTest.java
@@ -19,8 +19,10 @@ package org.apache.flink.streaming.api;
import static org.junit.Assert.assertEquals;
+import java.io.Serializable;
import java.util.ArrayList;
+import org.apache.flink.api.common.functions.CrossFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
@@ -29,7 +31,10 @@ import org.apache.flink.streaming.api.function.sink.SinkFunction;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.junit.Test;
-public class WindowCrossJoinTest {
+public class WindowCrossJoinTest implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
private static final long MEMORYSIZE = 32;
private static ArrayList<Tuple2<Tuple2<Integer, String>, Integer>> joinResults = new ArrayList<Tuple2<Tuple2<Integer, String>, Integer>>();
@@ -93,7 +98,17 @@ public class WindowCrossJoinTest {
inStream1.join(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
.where(0).equalTo(0).addSink(new JoinResultSink());
- inStream1.windowCross(inStream2, 1000, 1000, new MyTimestamp1(), new MyTimestamp2())
+ inStream1.cross(inStream2).onWindow(1000, 1000, new MyTimestamp1(), new MyTimestamp2())
+ .with(new CrossFunction<Tuple2<Integer,String>, Integer, Tuple2<Tuple2<Integer,String>, Integer>>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Tuple2<Tuple2<Integer, String>, Integer> cross(
+ Tuple2<Integer, String> val1, Integer val2) throws Exception {
+ return new Tuple2<Tuple2<Integer,String>, Integer>(val1, val2);
+ }
+ })
.addSink(new CrossResultSink());
env.executeTest(MEMORYSIZE);
[05/34] incubator-flink git commit: [streaming] Introduced a new
invokable to allow flexible windowing based on trigger and eviction policies.
Additionally created all required policy interfaces.
Posted by mb...@apache.org.
[streaming] Introduced a new invokable to allow flexible windowing based on trigger and eviction policies. Additionally created all required policy interfaces.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/36f1ee88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/36f1ee88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/36f1ee88
Branch: refs/heads/master
Commit: 36f1ee88cbadc21b185782026e6a6fc77d41b42a
Parents: a677c77
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:08:23 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../invokable/operator/WindowingInvokable.java | 284 +++++++++++++++++++
.../windowing/policy/ActiveEvictionPolicy.java | 52 ++++
.../windowing/policy/ActiveTriggerCallback.java | 48 ++++
.../windowing/policy/ActiveTriggerPolicy.java | 81 ++++++
.../api/windowing/policy/EvictionPolicy.java | 47 +++
.../api/windowing/policy/TriggerPolicy.java | 54 ++++
6 files changed, 566 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/36f1ee88/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
new file mode 100644
index 0000000..52b5edf
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.NoSuchElementException;
+
+public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, String[]>> {
+
+ /**
+ * Auto-generated serial version UID
+ */
+ private static final long serialVersionUID = -8038984294071650730L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(WindowingInvokable.class);
+
+ private LinkedList<TriggerPolicy<IN>> triggerPolicies;
+ private LinkedList<EvictionPolicy<IN>> evictionPolicies;
+ private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
+ private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
+ private LinkedList<Thread> activePolicyTreads = new LinkedList<Thread>();
+ private LinkedList<IN> buffer = new LinkedList<IN>();
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ private ReduceFunction<IN> reducer;
+
+ public WindowingInvokable(ReduceFunction<IN> userFunction,
+ LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction);
+
+ this.reducer = userFunction;
+ this.triggerPolicies = triggerPolicies;
+ this.evictionPolicies = evictionPolicies;
+
+ activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+ for (TriggerPolicy<IN> tp : triggerPolicies) {
+ if (tp instanceof ActiveTriggerPolicy) {
+ activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
+ }
+ }
+
+ activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+ for (EvictionPolicy<IN> ep : evictionPolicies) {
+ if (ep instanceof ActiveEvictionPolicy) {
+ activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
+ }
+ }
+ }
+
+ @Override
+ public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
+ super.open(parameters);
+ for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
+ Runnable target=tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+ if (target!=null){
+ Thread thread = new Thread(target);
+ activePolicyTreads.add(thread);
+ thread.start();
+ }
+ }
+ };
+
+ private class WindowingCallback implements ActiveTriggerCallback<IN> {
+ private ActiveTriggerPolicy<IN> policy;
+
+ public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+ this.policy = policy;
+ }
+
+ @Override
+ public void sendFakeElement(IN datapoint) {
+ processFakeElement(datapoint, this.policy);
+ }
+
+ }
+
+ @Override
+ protected void immutableInvoke() throws Exception {
+ // Prevent empty data streams
+ if ((reuse = recordIterator.next(reuse)) == null) {
+ throw new RuntimeException("DataStream must not be empty");
+ }
+
+ // Continuously run
+ while (reuse != null) {
+ processRealElement(reuse.getObject());
+
+ // Recreate the reuse-StremRecord object and load next StreamRecord
+ resetReuse();
+ reuse = recordIterator.next(reuse);
+ }
+
+ // Stop all remaining threads from policies
+ for (Thread t : activePolicyTreads) {
+ t.interrupt();
+ }
+
+ // finally trigger the buffer.
+ if (!buffer.isEmpty()) {
+ currentTriggerPolicies.clear();
+ for (TriggerPolicy<IN> policy : triggerPolicies) {
+ currentTriggerPolicies.add(policy);
+ }
+ callUserFunctionAndLogException();
+ }
+
+ }
+
+ @Override
+ protected void mutableInvoke() throws Exception {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("There is currently no mutable implementation of this operator. Immutable version is used.");
+ }
+ immutableInvoke();
+ }
+
+ /**
+ * This method processed an arrived fake element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowingInvokable#processRealElement(Object)}
+ *
+ * @param input
+ * a fake input element
+ * @param currentPolicy
+ * the policy which produced this fake element
+ */
+ private synchronized void processFakeElement(IN input, TriggerPolicy<IN> currentPolicy) {
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+
+ // Set the current trigger
+ currentTriggerPolicies.add(currentPolicy);
+
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+ }
+
+ /**
+ * This method processed an arrived real element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowingInvokable#processFakeElement(Object)}
+ *
+ * @param input
+ * a real input element
+ */
+ private synchronized void processRealElement(IN input) {
+ // Run the precalls to detect missed windows
+ for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
+ // Remark: In case multiple active triggers are present the ordering
+ // of the different fake elements returned by this triggers becomes
+ // a problem. This might lead to unexpected results...
+ // Should we limit the number of active triggers to 0 or 1?
+ IN[] result = trigger.preNotifyTrigger(input);
+ for (IN in : result) {
+ processFakeElement(in, trigger);
+ }
+ }
+
+ // Remember if a trigger occurred
+ boolean isTriggered = false;
+
+ // Process the triggers
+ for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
+ if (triggerPolicy.notifyTrigger(input)) {
+ currentTriggerPolicies.add(triggerPolicy);
+ }
+ }
+
+ // call user function
+ if (!currentTriggerPolicies.isEmpty()) {
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+
+ // remember trigger
+ isTriggered = true;
+ }
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+
+ for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+
+ // Add the current element to the buffer
+ buffer.add(input);
+
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ Iterator<IN> reducedIterator = buffer.iterator();
+ IN reduced = null;
+
+ while (reducedIterator.hasNext() && reduced == null) {
+ reduced = reducedIterator.next();
+ }
+
+ while (reducedIterator.hasNext()) {
+ IN next = reducedIterator.next();
+ if (next != null) {
+ reduced = reducer.reduce(reduced, next);
+ }
+ }
+ if (reduced != null) {
+ String[] tmp = new String[currentTriggerPolicies.size()];
+ for (int i = 0; i < tmp.length; i++) {
+ tmp[i] = currentTriggerPolicies.get(i).toString();
+ }
+ collector.collect(new Tuple2<IN, String[]>(reduced, tmp));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/36f1ee88/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
new file mode 100644
index 0000000..17d4914
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This interface is used for active eviction policies. beside the functionality
+ * inherited from {@link EvictionPolicy} it provides a method which gets called
+ * to notify on fake elements.
+ *
+ * In case an eviction policy implements this interface instead of the
+ * {@link EvictionPolicy} interface, not only the real but also fake data points
+ * will cause a notification of the eviction.
+ *
+ * Fake data points are mostly used in windowing based on time to trigger and
+ * evict even if no element arrives at all during a windows duration.
+ */
+public interface ActiveEvictionPolicy<DATA> extends EvictionPolicy<DATA> {
+
+ /**
+ * Proves if and how many elements should be deleted from the element
+ * buffer. The eviction takes place after the trigger and after the call to
+ * the UDF. This method is only called with fake elements.
+ *
+ * Note: Fake elements are always considered as triggered. Therefore this
+ * method does not have a triggered parameter.
+ *
+ * @param datapoint
+ * the current fake data point
+ * @param bufferSize
+ * the current size of the buffer (only real elements are
+ * counted)
+ * @return the number of elements to delete from the buffer (only real
+ * elements are counted)
+ */
+ public int notifyEvictionWithFakeElement(DATA datapoint, int bufferSize);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/36f1ee88/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
new file mode 100644
index 0000000..d74a3ae
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * In case an {@link ActiveTriggerPolicy} is used, it can implement own
+ * {@link Runnable} classes. Such {@link Runnable} classes will be executed as
+ * an own thread and can submit fake elements, to the element
+ * buffer at any time.
+ *
+ * The factory method for runnables of the {@link ActiveTriggerPolicy} gets an
+ * instance of this interface as parameter. The describes adding of elements can
+ * be done by the runnable using the methods provided in this interface.
+ *
+ * @param <DATA>
+ * The data type which can be consumed by the methods provided in
+ * this callback implementation.
+ */
+public interface ActiveTriggerCallback<DATA> {
+
+ /**
+ * Submits a new fake data point to the element buffer. Such a fake element
+ * might be used to trigger at any time, but will never be included in the
+ * result of the reduce function. The submission of a fake element causes
+ * notifications only at the {@link ActiveTriggerPolicy} and
+ * {@link ActiveEvictionPolicy} implementations.
+ *
+ * @param datapoint
+ * the fake data point to be added
+ */
+ public void sendFakeElement(DATA datapoint);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/36f1ee88/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
new file mode 100644
index 0000000..f549766
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+
+/**
+ * This interface extends the {@link TriggerPolicy} interface with functionality
+ * for active triggers. Active triggers can act in two ways:
+ *
+ * 1) Whenever an element arrives at the invokable, the
+ * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called first.
+ * It can return zero ore more fake data points which will be added before the
+ * the currently arrived real element gets processed. This allows to handle
+ * empty windows in time based windowing with an user defined {@link TimeStamp}.
+ * Triggers are not called on fake datapoint. A fake datapoint is always
+ * considered as triggered.
+ *
+ * 2) An active trigger has a factory method for a runnable. This factory method
+ * gets called at the start up of the invokable. The returned runnable will be
+ * executed in its own thread and can submit fake elements at any time threw an
+ * {@link ActiveTriggerCallback}. This allows to have time based triggers based
+ * on any system internal time measure. Triggers are not called on fake
+ * datapoint. A fake datapoints is always considered as triggered.
+ *
+ * @param <DATA>
+ * The data type which can be handled by this policy
+ */
+public interface ActiveTriggerPolicy<DATA> extends TriggerPolicy<DATA> {
+
+ /**
+ * Whenever an element arrives at the invokable, the
+ * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called
+ * first. It can return zero ore more fake data points which will be added
+ * before the the currently arrived real element gets processed. This allows
+ * to handle empty windows in time based windowing with an user defined
+ * {@link TimeStamp}. Triggers are not called on fake datapoints. A fake
+ * datapoint is always considered as triggered.
+ *
+ * @param datapoint
+ * the data point which arrived at the invokable
+ * @return zero ore more fake data points which will be added before the the
+ * currently arrived real element gets processed.
+ */
+ public DATA[] preNotifyTrigger(DATA datapoint);
+
+ /**
+ * This is the factory method for a runnable. This factory method gets
+ * called at the start up of the invokable. The returned runnable will be
+ * executed in its own thread and can submit fake elements at any time threw
+ * an {@link ActiveTriggerCallback}. This allows to have time based triggers
+ * based on any system internal time measure. Triggers are not called on
+ * fake datapoints. A fake datapoint is always considered as triggered.
+ *
+ * @param callback
+ * A callback object which allows to add fake elements from
+ * within the returned {@link Runnable}.
+ * @return The runnable implementation or null in case there is no. In case
+ * an {@link ActiveTriggerPolicy} is used, it can implement own
+ * {@link Runnable} classes. Such {@link Runnable} classes will be
+ * executed as an own thread and can submit fake elements, to the
+ * element buffer at any time.
+ */
+ public Runnable createActiveTriggerRunnable(ActiveTriggerCallback<DATA> callback);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/36f1ee88/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
new file mode 100644
index 0000000..c224ad4
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/EvictionPolicy.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.io.Serializable;
+
+/**
+ * An eviction policy specifies under which condition data points should be
+ * deleted from the buffer. Deletions must be done only in the order the
+ * elements arrived. Therefore, the policy only returns the number of elements
+ * to evict on each element arrival.
+ *
+ * @param <DATA>
+ * the type of the data handled by this policy
+ */
+public interface EvictionPolicy<DATA> extends Serializable {
+
+ /**
+ * Proves if and how many elements should be deleted from the element
+ * buffer. The eviction takes place after the trigger and after the call to
+ * the UDF but before the adding of the new data point.
+ *
+ * @param datapoint
+ * data point the data point which arrived
+ * @param triggered
+ * Information whether the UDF was triggered or not
+ * @param bufferSize
+ * The current size of the element buffer at the invokable
+ * @return The number of elements to be deleted from the buffer
+ */
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize);
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/36f1ee88/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
new file mode 100644
index 0000000..c212df6
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TriggerPolicy.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.io.Serializable;
+
+/**
+ * Proves and returns if a new window should be started. In case the trigger
+ * occurs (return value true) the UDF will be executed on the current element
+ * buffer without the last added element which is provided as parameter. This
+ * element will be added to the buffer after the execution of the UDF.
+ *
+ * @param <DATA>
+ * The data type which can be handled by this policy
+ */
+public interface TriggerPolicy<DATA> extends Serializable {
+
+ /**
+ * Proves and returns if a new window should be started. In case the trigger
+ * occurs (return value true) the UDF will be executed on the current
+ * element buffer without the last added element which is provided as
+ * parameter. This element will be added to the buffer after the execution
+ * of the UDF.
+ *
+ * There are possibly different strategies for eviction and triggering: 1)
+ * including last data point: Better/faster for count eviction 2) excluding
+ * last data point: Essentially required for time based eviction and delta
+ * rules As 2) is required for some policies and the benefit of using 1) is
+ * small for the others, policies are implemented according to 2).
+ *
+ * @param datapoint
+ * the data point which arrived
+ * @return true if the current windows should be closed, otherwise false. In
+ * true case the given data point will be part of the next window
+ * and will not be included in the current one.
+ */
+ public boolean notifyTrigger(DATA datapoint);
+
+}
[25/34] incubator-flink git commit: [streaming] Make all predefined
policies cloneable and introduce an activating wrapper for cloneable eviction
policies
Posted by mb...@apache.org.
[streaming] Make all predefined policies cloneable and introduce an activating wrapper for cloneable eviction policies
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/70079c1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/70079c1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/70079c1d
Branch: refs/heads/master
Commit: 70079c1d49fb81336c8846e4058301bd1b6b6b8e
Parents: cd7bfd3
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Thu Nov 20 14:45:04 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../ActiveCloneableEvictionPolicyWrapper.java | 62 ++++++++++++++++++++
.../windowing/policy/CountEvictionPolicy.java | 6 +-
.../windowing/policy/CountTriggerPolicy.java | 6 +-
.../api/windowing/policy/DeltaPolicy.java | 7 ++-
.../api/windowing/policy/PunctuationPolicy.java | 10 +++-
.../windowing/policy/TimeEvictionPolicy.java | 8 ++-
.../api/windowing/policy/TimeTriggerPolicy.java | 12 +++-
.../policy/TumblingEvictionPolicy.java | 7 ++-
8 files changed, 108 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
new file mode 100644
index 0000000..29ba9eb
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveCloneableEvictionPolicyWrapper.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * The {@link ActiveEvictionPolicy} wraps around a non active
+ * {@link EvictionPolicy}. It forwards all calls to
+ * {@link ActiveEvictionPolicy#notifyEvictionWithFakeElement(Object, int)} to
+ * {@link EvictionPolicy#notifyEviction(Object, boolean, int)} while the
+ * triggered parameter will be set to true.
+ *
+ * This class additionally implements the clone method and can wrap around
+ * {@link CloneableEvictionPolicy} to make it active.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public class ActiveCloneableEvictionPolicyWrapper<DATA> extends ActiveEvictionPolicyWrapper<DATA>
+ implements CloneableEvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = 1520261575300622769L;
+ CloneableEvictionPolicy<DATA> nestedPolicy;
+
+ /**
+ * Creates a wrapper which activates the eviction policy which is wrapped
+ * in. This means that the nested policy will get called on fake elements as
+ * well as on real elements.
+ *
+ * This specialized version of the {@link ActiveEvictionPolicyWrapper} works
+ * with {@link CloneableEvictionPolicy} and is thereby cloneable as well.
+ *
+ * @param nestedPolicy
+ * The policy which should be activated/wrapped in.
+ */
+ public ActiveCloneableEvictionPolicyWrapper(CloneableEvictionPolicy<DATA> nestedPolicy) {
+ super(nestedPolicy);
+ this.nestedPolicy = nestedPolicy;
+ }
+
+ @Override
+ public ActiveCloneableEvictionPolicyWrapper<DATA> clone() {
+ return new ActiveCloneableEvictionPolicyWrapper<DATA>(nestedPolicy.clone());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
index 1d8175c..a9e7b3f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
@@ -27,7 +27,7 @@ package org.apache.flink.streaming.api.windowing.policy;
* @param <IN>
* the type of the incoming data points
*/
-public class CountEvictionPolicy<IN> implements EvictionPolicy<IN> {
+public class CountEvictionPolicy<IN> implements CloneableEvictionPolicy<IN> {
/**
* Auto generated version id
@@ -112,4 +112,8 @@ public class CountEvictionPolicy<IN> implements EvictionPolicy<IN> {
}
}
+ @Override
+ public CountEvictionPolicy<IN> clone() {
+ return new CountEvictionPolicy<IN>(maxElements, deleteOnEviction, counter);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
index ca0058e..d5ae932 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
@@ -23,7 +23,7 @@ package org.apache.flink.streaming.api.windowing.policy;
* @param <IN>
* The type of the data points which are handled by this policy
*/
-public class CountTriggerPolicy<IN> implements TriggerPolicy<IN> {
+public class CountTriggerPolicy<IN> implements CloneableTriggerPolicy<IN> {
/**
* Auto generated version ID
@@ -80,4 +80,8 @@ public class CountTriggerPolicy<IN> implements TriggerPolicy<IN> {
}
}
+ @Override
+ public CountTriggerPolicy<IN> clone() {
+ return new CountTriggerPolicy<IN>(max, counter);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
index 418f989..77bc692 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
@@ -39,7 +39,8 @@ import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
* @param <DATA>
* The type of the data points which are handled by this policy
*/
-public class DeltaPolicy<DATA> implements TriggerPolicy<DATA>, EvictionPolicy<DATA> {
+public class DeltaPolicy<DATA> implements CloneableTriggerPolicy<DATA>,
+ CloneableEvictionPolicy<DATA> {
/**
* Auto generated version ID
@@ -104,4 +105,8 @@ public class DeltaPolicy<DATA> implements TriggerPolicy<DATA>, EvictionPolicy<DA
return evictCount;
}
+ @Override
+ public DeltaPolicy<DATA> clone() {
+ return new DeltaPolicy<DATA>(deltaFuntion, triggerDataPoint, threshold);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
index 0ec9a57..ee8a3cb 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
@@ -28,7 +28,8 @@ import org.apache.flink.streaming.api.windowing.extractor.Extractor;
* deleted in case the punctuation is detected.
*
* By default this policy does not react on fake elements. Wrap it in an
- * {@link ActiveEvictionPolicyWrapper} to make it react on punctuation even in fake elements.
+ * {@link ActiveEvictionPolicyWrapper} to make it react on punctuation even in
+ * fake elements.
*
* @param <IN>
* The type of the input data handled by this policy. An
@@ -37,7 +38,8 @@ import org.apache.flink.streaming.api.windowing.extractor.Extractor;
* The type of the punctuation. An {@link Extractor} can be used to
* extract DATA for IN.
*/
-public class PunctuationPolicy<IN, DATA> implements TriggerPolicy<IN>, EvictionPolicy<IN> {
+public class PunctuationPolicy<IN, DATA> implements CloneableTriggerPolicy<IN>,
+ CloneableEvictionPolicy<IN> {
/**
* auto generated version id
@@ -107,4 +109,8 @@ public class PunctuationPolicy<IN, DATA> implements TriggerPolicy<IN>, EvictionP
}
}
+ @Override
+ public PunctuationPolicy<IN, DATA> clone() {
+ return new PunctuationPolicy<IN, DATA>(punctuation, extractor);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
index a78cc4e..d9acc84 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
@@ -31,7 +31,8 @@ import org.apache.flink.streaming.api.invokable.util.TimeStamp;
* The type of the incoming data points which are processed by this
* policy.
*/
-public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA> {
+public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
+ CloneableEvictionPolicy<DATA> {
/**
* auto generated version id
@@ -106,4 +107,9 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA> {
return counter;
}
+ @Override
+ public TimeEvictionPolicy<DATA> clone() {
+ return new TimeEvictionPolicy<DATA>(granularity, timestamp);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
index d50e285..ff93ac9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -33,7 +33,8 @@ import org.apache.flink.streaming.api.windowing.extractor.Extractor;
* The type of the incoming data points which are processed by this
* policy.
*/
-public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA> {
+public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
+ CloneableTriggerPolicy<DATA> {
/**
* auto generated version id
@@ -115,8 +116,8 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA> {
}
/**
- * This method checks if we missed a window end. If this is the
- * case we trigger the missed windows using fake elements.
+ * This method checks if we missed a window end. If this is the case we
+ * trigger the missed windows using fake elements.
*/
@SuppressWarnings("unchecked")
@Override
@@ -190,4 +191,9 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA> {
}
}
+ @Override
+ public TimeTriggerPolicy<DATA> clone() {
+ return new TimeTriggerPolicy<DATA>(granularity, timestamp, 0, longToDATAExtractor);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/70079c1d/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
index dc7f5b0..9a2e3c9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TumblingEvictionPolicy.java
@@ -29,7 +29,7 @@ package org.apache.flink.streaming.api.windowing.policy;
* @param <DATA>
* The type of the data points which is handled by this policy
*/
-public class TumblingEvictionPolicy<DATA> implements EvictionPolicy<DATA> {
+public class TumblingEvictionPolicy<DATA> implements CloneableEvictionPolicy<DATA> {
/**
* Auto generated version ID
@@ -82,4 +82,9 @@ public class TumblingEvictionPolicy<DATA> implements EvictionPolicy<DATA> {
return 0;
}
}
+
+ @Override
+ public TumblingEvictionPolicy<DATA> clone() {
+ return new TumblingEvictionPolicy<DATA>(counter);
+ }
}
[18/34] incubator-flink git commit: [streaming] Windowing API update
+ package refactor
Posted by mb...@apache.org.
[streaming] Windowing API update + package refactor
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/2d73c3f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/2d73c3f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/2d73c3f1
Branch: refs/heads/master
Commit: 2d73c3f168b4006f3100941dc8121c4e4c534c48
Parents: 55ee64f
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Nov 24 19:44:01 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
docs/streaming_guide.md | 2 +-
.../streaming/api/datastream/DataStream.java | 722 +++++++++----------
.../api/datastream/WindowedDataStream.java | 255 ++++---
.../operator/GroupedWindowInvokable.java | 322 +++++++++
.../operator/GroupedWindowingInvokable.java | 322 ---------
.../operator/WindowGroupReduceInvokable.java | 43 ++
.../api/invokable/operator/WindowInvokable.java | 368 ++++++++++
.../operator/WindowReduceInvokable.java | 59 ++
.../operator/WindowingGroupInvokable.java | 43 --
.../invokable/operator/WindowingInvokable.java | 368 ----------
.../operator/WindowingReduceInvokable.java | 59 --
.../policy/CloneableEvictionPolicy.java | 4 +-
.../policy/CloneableTriggerPolicy.java | 4 +-
.../operator/GroupedWindowingInvokableTest.java | 10 +-
.../operator/WindowingInvokableTest.java | 8 +-
15 files changed, 1315 insertions(+), 1274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index a8002ff..a0bc1d6 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -257,7 +257,7 @@ The Flink Streaming API supports different types of aggregation operators simila
Types of aggregations: `sum(fieldPosition)`, `min(fieldPosition)`, `max(fieldPosition)`, `minBy(fieldPosition, first)`, `maxBy(fieldPosition, first)`
-With `sum`, `min`, and `max` for every incoming tuple the selected field is replaced with the current aggregated value. If the aggregations are used without defining field position, position `0` is used as default.
+With `sum`, `min`, and `max` for every incoming tuple the selected field is replaced with the current aggregated value.
With `minBy` and `maxBy` the output of the operator is the element with the current minimal or maximal value at the given fieldposition. If more components share the minimum or maximum value, the user can decide if the operator should return the first or last element. This can be set by the `first` boolean parameter.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 2ce28d9..49981a5 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -64,6 +64,8 @@ import org.apache.flink.streaming.api.windowing.helper.Count;
import org.apache.flink.streaming.api.windowing.helper.Delta;
import org.apache.flink.streaming.api.windowing.helper.Time;
import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.partitioner.BroadcastPartitioner;
import org.apache.flink.streaming.partitioner.DistributePartitioner;
import org.apache.flink.streaming.partitioner.FieldsPartitioner;
@@ -194,67 +196,6 @@ public class DataStream<OUT> {
}
/**
- * Gets the class of the field at the given position
- *
- * @param pos
- * Position of the field
- * @return The class of the field
- */
- @SuppressWarnings("rawtypes")
- protected Class<?> getClassAtPos(int pos) {
- Class<?> type;
- TypeInformation<OUT> outTypeInfo = outTypeWrapper.getTypeInfo();
- if (outTypeInfo.isTupleType()) {
- type = ((TupleTypeInfo) outTypeInfo).getTypeAt(pos).getTypeClass();
-
- } else if (outTypeInfo instanceof BasicArrayTypeInfo) {
-
- type = ((BasicArrayTypeInfo) outTypeInfo).getComponentTypeClass();
-
- } else if (outTypeInfo instanceof PrimitiveArrayTypeInfo) {
- Class<?> clazz = outTypeInfo.getTypeClass();
- if (clazz == boolean[].class) {
- type = Boolean.class;
- } else if (clazz == short[].class) {
- type = Short.class;
- } else if (clazz == int[].class) {
- type = Integer.class;
- } else if (clazz == long[].class) {
- type = Long.class;
- } else if (clazz == float[].class) {
- type = Float.class;
- } else if (clazz == double[].class) {
- type = Double.class;
- } else if (clazz == char[].class) {
- type = Character.class;
- } else {
- throw new IndexOutOfBoundsException("Type could not be determined for array");
- }
-
- } else if (pos == 0) {
- type = outTypeInfo.getTypeClass();
- } else {
- throw new IndexOutOfBoundsException("Position is out of range");
- }
- return type;
- }
-
- /**
- * Checks if the given field position is allowed for the output type
- *
- * @param pos
- * Position to check
- */
- protected void checkFieldRange(int pos) {
- try {
- getClassAtPos(pos);
- } catch (IndexOutOfBoundsException e) {
- throw new RuntimeException("Selected field is out of range");
-
- }
- }
-
- /**
* Creates a new {@link DataStream} by merging {@link DataStream} outputs of
* the same type with each other. The DataStreams merged using this operator
* will be transformed simultaneously.
@@ -275,14 +216,6 @@ public class DataStream<OUT> {
return returnStream;
}
- private void validateMerge(String id) {
- for (DataStream<OUT> ds : this.mergedStreams) {
- if (ds.getId().equals(id)) {
- throw new RuntimeException("A DataStream cannot be merged with itself");
- }
- }
- }
-
/**
* Creates a new {@link ConnectedDataStream} by connecting
* {@link DataStream} outputs of different type with each other. The
@@ -297,152 +230,51 @@ public class DataStream<OUT> {
}
/**
- * Creates a cross (Cartesian product) of a data stream window. The user can
- * implement their own time stamps or use the system time by default.
- *
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param dataStreamToCross
- * @param windowSize
- * @param slideInterval
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowCross(
- DataStream<IN2> dataStreamToCross, long windowSize, long slideInterval) {
- return this.windowCross(dataStreamToCross, windowSize, slideInterval,
- new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>());
- }
-
- /**
- * Creates a cross (Cartesian product) of a data stream window.
- *
- * @param dataStreamToCross
- * {@link DataStream} to cross with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param timestamp1
- * User defined time stamps for the first input.
- * @param timestamp2
- * User defined time stamps for the second input.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowCross(
- DataStream<IN2> dataStreamToCross, long windowSize, long slideInterval,
- TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2) {
- return this.connect(dataStreamToCross).windowCross(windowSize, slideInterval, timestamp1,
- timestamp2);
- }
-
- /**
- * Creates a join of a data stream based on the given positions.
- *
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval, int fieldIn1,
- int fieldIn2) {
- return this.windowJoin(dataStreamToJoin, windowSize, slideInterval,
- new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
- }
-
- /**
- * Creates a join of a data stream based on the given positions.
+ * Groups the elements of a {@link DataStream} by the given key positions to
+ * be used with grouped operators like
+ * {@link GroupedDataStream#reduce(ReduceFunction)}
*
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
+ * @param fields
+ * The position of the fields on which the {@link DataStream}
+ * will be grouped.
+ * @return The grouped {@link DataStream}
*/
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval, String fieldIn1,
- String fieldIn2) {
- return this.windowJoin(dataStreamToJoin, windowSize, slideInterval,
- new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
+ public GroupedDataStream<OUT> groupBy(int... fields) {
+
+ return groupBy(FieldsKeySelector.getSelector(getOutputType(), fields));
+
}
/**
- * Creates a join of a data stream based on the given positions.
+ * Groups a {@link DataStream} using field expressions. A field expression
+ * is either the name of a public field or a getter method with parentheses
+ * of the {@link DataStream}S underlying type. A dot can be used to drill
+ * down into objects, as in {@code "field1.getInnerField2()" }. This method
+ * returns an {@link GroupedDataStream}.
*
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param timestamp1
- * User defined time stamps for the first input.
- * @param timestamp2
- * User defined time stamps for the second input.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
- */
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval,
- TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2, int fieldIn1, int fieldIn2) {
- return this.connect(dataStreamToJoin).windowJoin(windowSize, slideInterval, timestamp1,
- timestamp2, fieldIn1, fieldIn2);
+ * @param fields
+ * One or more field expressions on which the DataStream will be
+ * grouped.
+ * @return The grouped {@link DataStream}
+ **/
+ public GroupedDataStream<OUT> groupBy(String... fields) {
+
+ return groupBy(new PojoKeySelector<OUT>(getOutputType(), fields));
+
}
/**
- * Creates a join of a data stream based on the given positions.
+ * Groups the elements of a {@link DataStream} by the key extracted by the
+ * {@link KeySelector} to be used with grouped operators like
+ * {@link GroupedDataStream#reduce(ReduceFunction)}
*
- * @param dataStreamToJoin
- * {@link DataStream} to join with.
- * @param windowSize
- * Size of the windows that will be aligned for both streams in
- * milliseconds.
- * @param slideInterval
- * After every function call the windows will be slid by this
- * interval.
- * @param timestamp1
- * User defined time stamps for the first input.
- * @param timestamp2
- * User defined time stamps for the second input.
- * @param fieldIn1
- * The field in the first stream to be matched.
- * @param fieldIn2
- * The field in the second stream to be matched.
- * @return The transformed {@link DataStream}.
+ * @param keySelector
+ * The {@link KeySelector} that will be used to extract keys for
+ * the values
+ * @return The grouped {@link DataStream}
*/
- public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
- DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval,
- TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2, String fieldIn1, String fieldIn2) {
- return this.connect(dataStreamToJoin).windowJoin(windowSize, slideInterval, timestamp1,
- timestamp2, fieldIn1, fieldIn2);
+ public GroupedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
+ return new GroupedDataStream<OUT>(this, keySelector);
}
/**
@@ -454,7 +286,7 @@ public class DataStream<OUT> {
* @return The DataStream with fields partitioning set.
*/
public DataStream<OUT> partitionBy(int... fields) {
-
+
return setConnectionType(new FieldsPartitioner<OUT>(FieldsKeySelector.getSelector(
getOutputType(), fields)));
}
@@ -468,7 +300,7 @@ public class DataStream<OUT> {
* @return The DataStream with fields partitioning set.
*/
public DataStream<OUT> partitionBy(String... fields) {
-
+
return setConnectionType(new FieldsPartitioner<OUT>(new PojoKeySelector<OUT>(
getOutputType(), fields)));
}
@@ -526,6 +358,32 @@ public class DataStream<OUT> {
}
/**
+ * Initiates an iterative part of the program that executes multiple times
+ * and feeds back data streams. The iterative part needs to be closed by
+ * calling {@link IterativeDataStream#closeWith(DataStream)}. The
+ * transformation of this IterativeDataStream will be the iteration head.
+ * The data stream given to the {@code closeWith(DataStream)} method is the
+ * data stream that will be fed back and used as the input for the iteration
+ * head. Unlike in batch processing by default the output of the iteration
+ * stream is directed to both to the iteration head and the next component.
+ * To direct tuples to the iteration head or the output specifically one can
+ * use the {@code split(OutputSelector)} on the iteration tail while
+ * referencing the iteration head as 'iterate'.
+ * <p>
+ * The iteration edge will be partitioned the same way as the first input of
+ * the iteration head.
+ * <p>
+ * By default a DataStream with iteration will never terminate, but the user
+ * can use the {@link IterativeDataStream#setMaxWaitTime} call to set a max
+ * waiting time for the iteration.
+ *
+ * @return The iterative data stream created.
+ */
+ public IterativeDataStream<OUT> iterate() {
+ return new IterativeDataStream<OUT>(this);
+ }
+
+ /**
* Applies a Map transformation on a {@link DataStream}. The transformation
* calls a {@link MapFunction} for each element of the DataStream. Each
* MapFunction call returns exactly one element. The user can also extend
@@ -593,6 +451,28 @@ public class DataStream<OUT> {
}
/**
+ * Applies a Filter transformation on a {@link DataStream}. The
+ * transformation calls a {@link FilterFunction} for each element of the
+ * DataStream and retains only those element for which the function returns
+ * true. Elements for which the function returns false are filtered. The
+ * user can also extend {@link RichFilterFunction} to gain access to other
+ * features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param filter
+ * The FilterFunction that is called for each element of the
+ * DataSet.
+ * @return The filtered DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> filter(FilterFunction<OUT> filter) {
+ FunctionTypeWrapper<OUT> typeWrapper = new FunctionTypeWrapper<OUT>(filter,
+ FilterFunction.class, 0);
+
+ return addFunction("filter", filter, typeWrapper, typeWrapper, new FilterInvokable<OUT>(
+ filter));
+ }
+
+ /**
* Initiates a Project transformation on a {@link Tuple} {@link DataStream}.<br/>
* <b>Note: Only Tuple DataStreams can be projected.</b></br> The
* transformation projects each Tuple of the DataSet onto a (sub)set of
@@ -616,69 +496,6 @@ public class DataStream<OUT> {
}
/**
- * Groups the elements of a {@link DataStream} by the given key positions to
- * be used with grouped operators like
- * {@link GroupedDataStream#reduce(ReduceFunction)}
- *
- * @param fields
- * The position of the fields on which the {@link DataStream}
- * will be grouped.
- * @return The grouped {@link DataStream}
- */
- public GroupedDataStream<OUT> groupBy(int... fields) {
-
- return groupBy(FieldsKeySelector.getSelector(getOutputType(), fields));
-
- }
-
- /**
- * Groups a {@link DataStream} using field expressions. A field expression
- * is either the name of a public field or a getter method with parentheses
- * of the {@link DataStream}S underlying type. A dot can be used to drill
- * down into objects, as in {@code "field1.getInnerField2()" }. This method
- * returns an {@link GroupedDataStream}.
- *
- * @param fields
- * One or more field expressions on which the DataStream will be
- * grouped.
- * @return The grouped {@link DataStream}
- **/
- public GroupedDataStream<OUT> groupBy(String... fields) {
-
- return groupBy(new PojoKeySelector<OUT>(getOutputType(), fields));
-
- }
-
- /**
- * Groups the elements of a {@link DataStream} by the key extracted by the
- * {@link KeySelector} to be used with grouped operators like
- * {@link GroupedDataStream#reduce(ReduceFunction)}
- *
- * @param keySelector
- * The {@link KeySelector} that will be used to extract keys for
- * the values
- * @return The grouped {@link DataStream}
- */
- public GroupedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
- return new GroupedDataStream<OUT>(this, keySelector);
- }
-
- /**
- * This allows you to set up windowing through a nice API using
- * {@link WindowingHelper} such as {@link Time}, {@link Count} and
- * {@link Delta}.
- *
- * @param policyHelpers
- * Any {@link WindowingHelper} such as {@link Time},
- * {@link Count} and {@link Delta}.
- * @return A {@link WindowedDataStream} providing further operations.
- */
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public WindowedDataStream<OUT> window(WindowingHelper... policyHelpers) {
- return new WindowedDataStream<OUT>(this, policyHelpers);
- }
-
- /**
* Applies an aggregation that sums the data stream at the given position.
*
* @param positionToSum
@@ -709,15 +526,6 @@ public class DataStream<OUT> {
}
/**
- * Syntactic sugar for sum(0)
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum() {
- return sum(0);
- }
-
- /**
* Applies an aggregation that that gives the minimum of the data stream at
* the given position.
*
@@ -839,15 +647,6 @@ public class DataStream<OUT> {
}
/**
- * Syntactic sugar for min(0)
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min() {
- return min(0);
- }
-
- /**
* Applies an aggregation that gives the maximum of the data stream at the
* given position.
*
@@ -876,76 +675,222 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the current element with the
- * maximum value at the given position, if more elements have the maximum
- * value at the given position, the operator returns either the first or
- * last one, depending on the parameter set.
+ * Applies an aggregation that that gives the current element with the
+ * maximum value at the given position, if more elements have the maximum
+ * value at the given position, the operator returns either the first or
+ * last one, depending on the parameter set.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize.
+ * @param first
+ * If true, then the operator return the first element with the
+ * maximum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
+ checkFieldRange(positionToMaxBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getOutputType(),
+ AggregationType.MAXBY, first));
+ }
+
+ /**
+ * Applies an aggregation that gives the count of the values.
+ *
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<Long, ?> count() {
+ TypeWrapper<OUT> inTypeWrapper = outTypeWrapper;
+ TypeWrapper<Long> outTypeWrapper = new ObjectTypeWrapper<Long>(Long.valueOf(0));
+
+ return addFunction("counter", null, inTypeWrapper, outTypeWrapper,
+ new CounterInvokable<OUT>());
+ }
+
+ /**
+ * This allows you to set up windowing through a nice API using
+ * {@link WindowingHelper} such as {@link Time}, {@link Count} and
+ * {@link Delta}. Windowing allows the user to apply different user defined
+ * functions on predefined chunks of the data stream. For example a reducer
+ * could be applied on every 5 seconds of data to count the elements in that
+ * time window.
+ *
+ * @param policyHelpers
+ * Any {@link WindowingHelper} such as {@link Time},
+ * {@link Count} and {@link Delta}.
+ * @return A {@link WindowedDataStream} providing further operations.
+ */
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public WindowedDataStream<OUT> window(WindowingHelper... policyHelpers) {
+ return new WindowedDataStream<OUT>(this, policyHelpers);
+ }
+
+ /**
+ * Set up a windowed data stream using the given {@link TriggerPolicy}s and
+ * {@link EvictionPolicy}s. Windowing allows the user to apply different
+ * user defined functions on predefined chunks of the data stream. For
+ * example a reducer could be applied on every 5 seconds of data to count
+ * the elements in that time window.
+ *
+ * @param triggers
+ * The list of {@link TriggerPolicy}s that will determine how
+ * often the user function is called on the window.
+ * @param evicters
+ * The list of {@link EvictionPolicy}s that will determine the
+ * number of elements in each time window.
+ * @return A {@link WindowedDataStream} providing further operations.
+ */
+ public WindowedDataStream<OUT> window(List<TriggerPolicy<OUT>> triggers,
+ List<EvictionPolicy<OUT>> evicters) {
+ return new WindowedDataStream<OUT>(this, triggers, evicters);
+ }
+
+ /**
+ * Creates a cross (Cartesian product) of a data stream window. The user can
+ * implement their own time stamps or use the system time by default.
+ *
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ * @param dataStreamToCross
+ * @param windowSize
+ * @param slideInterval
+ * @return The transformed {@link DataStream}.
+ */
+ public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowCross(
+ DataStream<IN2> dataStreamToCross, long windowSize, long slideInterval) {
+ return this.windowCross(dataStreamToCross, windowSize, slideInterval,
+ new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>());
+ }
+
+ /**
+ * Creates a cross (Cartesian product) of a data stream window.
*
- * @param positionToMaxBy
- * The position in the data point to maximize.
- * @param first
- * If true, then the operator return the first element with the
- * maximum value, otherwise returns the last
- * @return The transformed DataStream.
+ * @param dataStreamToCross
+ * {@link DataStream} to cross with.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ * @param timestamp1
+ * User defined time stamps for the first input.
+ * @param timestamp2
+ * User defined time stamps for the second input.
+ * @return The transformed {@link DataStream}.
*/
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
- checkFieldRange(positionToMaxBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getOutputType(),
- AggregationType.MAXBY, first));
+ public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowCross(
+ DataStream<IN2> dataStreamToCross, long windowSize, long slideInterval,
+ TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2) {
+ return this.connect(dataStreamToCross).windowCross(windowSize, slideInterval, timestamp1,
+ timestamp2);
}
/**
- * Syntactic sugar for max(0)
+ * Creates a join of a data stream based on the given positions.
*
- * @return The transformed DataStream.
+ * @param dataStreamToJoin
+ * {@link DataStream} to join with.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ * @param fieldIn1
+ * The field in the first stream to be matched.
+ * @param fieldIn2
+ * The field in the second stream to be matched.
+ * @return The transformed {@link DataStream}.
*/
- public SingleOutputStreamOperator<OUT, ?> max() {
- return max(0);
+ public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
+ DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval, int fieldIn1,
+ int fieldIn2) {
+ return this.windowJoin(dataStreamToJoin, windowSize, slideInterval,
+ new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
}
/**
- * Applies an aggregation that gives the count of the values.
+ * Creates a join of a data stream based on the given positions.
*
- * @return The transformed DataStream.
+ * @param dataStreamToJoin
+ * {@link DataStream} to join with.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ * @param fieldIn1
+ * The field in the first stream to be matched.
+ * @param fieldIn2
+ * The field in the second stream to be matched.
+ * @return The transformed {@link DataStream}.
*/
- public SingleOutputStreamOperator<Long, ?> count() {
- TypeWrapper<OUT> inTypeWrapper = outTypeWrapper;
- TypeWrapper<Long> outTypeWrapper = new ObjectTypeWrapper<Long>(Long.valueOf(0));
-
- return addFunction("counter", null, inTypeWrapper, outTypeWrapper,
- new CounterInvokable<OUT>());
+ public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
+ DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval, String fieldIn1,
+ String fieldIn2) {
+ return this.windowJoin(dataStreamToJoin, windowSize, slideInterval,
+ new DefaultTimeStamp<OUT>(), new DefaultTimeStamp<IN2>(), fieldIn1, fieldIn2);
}
- protected SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
-
- StreamReduceInvokable<OUT> invokable = new StreamReduceInvokable<OUT>(aggregate);
-
- SingleOutputStreamOperator<OUT, ?> returnStream = addFunction("reduce", aggregate,
- outTypeWrapper, outTypeWrapper, invokable);
-
- return returnStream;
+ /**
+ * Creates a join of a data stream based on the given positions.
+ *
+ * @param dataStreamToJoin
+ * {@link DataStream} to join with.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ * @param timestamp1
+ * User defined time stamps for the first input.
+ * @param timestamp2
+ * User defined time stamps for the second input.
+ * @param fieldIn1
+ * The field in the first stream to be matched.
+ * @param fieldIn2
+ * The field in the second stream to be matched.
+ * @return The transformed {@link DataStream}.
+ */
+ public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
+ DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval,
+ TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2, int fieldIn1, int fieldIn2) {
+ return this.connect(dataStreamToJoin).windowJoin(windowSize, slideInterval, timestamp1,
+ timestamp2, fieldIn1, fieldIn2);
}
/**
- * Applies a Filter transformation on a {@link DataStream}. The
- * transformation calls a {@link FilterFunction} for each element of the
- * DataStream and retains only those element for which the function returns
- * true. Elements for which the function returns false are filtered. The
- * user can also extend {@link RichFilterFunction} to gain access to other
- * features provided by the
- * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ * Creates a join of a data stream based on the given positions.
*
- * @param filter
- * The FilterFunction that is called for each element of the
- * DataSet.
- * @return The filtered DataStream.
+ * @param dataStreamToJoin
+ * {@link DataStream} to join with.
+ * @param windowSize
+ * Size of the windows that will be aligned for both streams in
+ * milliseconds.
+ * @param slideInterval
+ * After every function call the windows will be slid by this
+ * interval.
+ * @param timestamp1
+ * User defined time stamps for the first input.
+ * @param timestamp2
+ * User defined time stamps for the second input.
+ * @param fieldIn1
+ * The field in the first stream to be matched.
+ * @param fieldIn2
+ * The field in the second stream to be matched.
+ * @return The transformed {@link DataStream}.
*/
- public SingleOutputStreamOperator<OUT, ?> filter(FilterFunction<OUT> filter) {
- FunctionTypeWrapper<OUT> typeWrapper = new FunctionTypeWrapper<OUT>(filter,
- FilterFunction.class, 0);
-
- return addFunction("filter", filter, typeWrapper, typeWrapper, new FilterInvokable<OUT>(
- filter));
+ public <IN2> SingleOutputStreamOperator<Tuple2<OUT, IN2>, ?> windowJoin(
+ DataStream<IN2> dataStreamToJoin, long windowSize, long slideInterval,
+ TimeStamp<OUT> timestamp1, TimeStamp<IN2> timestamp2, String fieldIn1, String fieldIn2) {
+ return this.connect(dataStreamToJoin).windowJoin(windowSize, slideInterval, timestamp1,
+ timestamp2, fieldIn1, fieldIn2);
}
/**
@@ -1262,30 +1207,14 @@ public class DataStream<OUT> {
return returnStream;
}
- /**
- * Initiates an iterative part of the program that executes multiple times
- * and feeds back data streams. The iterative part needs to be closed by
- * calling {@link IterativeDataStream#closeWith(DataStream)}. The
- * transformation of this IterativeDataStream will be the iteration head.
- * The data stream given to the {@code closeWith(DataStream)} method is the
- * data stream that will be fed back and used as the input for the iteration
- * head. Unlike in batch processing by default the output of the iteration
- * stream is directed to both to the iteration head and the next component.
- * To direct tuples to the iteration head or the output specifically one can
- * use the {@code split(OutputSelector)} on the iteration tail while
- * referencing the iteration head as 'iterate'.
- * <p>
- * The iteration edge will be partitioned the same way as the first input of
- * the iteration head.
- * <p>
- * By default a DataStream with iteration will never terminate, but the user
- * can use the {@link IterativeDataStream#setMaxWaitTime} call to set a max
- * waiting time for the iteration.
- *
- * @return The iterative data stream created.
- */
- public IterativeDataStream<OUT> iterate() {
- return new IterativeDataStream<OUT>(this);
+ protected SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
+
+ StreamReduceInvokable<OUT> invokable = new StreamReduceInvokable<OUT>(aggregate);
+
+ SingleOutputStreamOperator<OUT, ?> returnStream = addFunction("reduce", aggregate,
+ outTypeWrapper, outTypeWrapper, invokable);
+
+ return returnStream;
}
protected <R> DataStream<OUT> addIterationSource(String iterationID, long waitTime) {
@@ -1414,6 +1343,75 @@ public class DataStream<OUT> {
}
/**
+ * Gets the class of the field at the given position
+ *
+ * @param pos
+ * Position of the field
+ * @return The class of the field
+ */
+ @SuppressWarnings("rawtypes")
+ protected Class<?> getClassAtPos(int pos) {
+ Class<?> type;
+ TypeInformation<OUT> outTypeInfo = outTypeWrapper.getTypeInfo();
+ if (outTypeInfo.isTupleType()) {
+ type = ((TupleTypeInfo) outTypeInfo).getTypeAt(pos).getTypeClass();
+
+ } else if (outTypeInfo instanceof BasicArrayTypeInfo) {
+
+ type = ((BasicArrayTypeInfo) outTypeInfo).getComponentTypeClass();
+
+ } else if (outTypeInfo instanceof PrimitiveArrayTypeInfo) {
+ Class<?> clazz = outTypeInfo.getTypeClass();
+ if (clazz == boolean[].class) {
+ type = Boolean.class;
+ } else if (clazz == short[].class) {
+ type = Short.class;
+ } else if (clazz == int[].class) {
+ type = Integer.class;
+ } else if (clazz == long[].class) {
+ type = Long.class;
+ } else if (clazz == float[].class) {
+ type = Float.class;
+ } else if (clazz == double[].class) {
+ type = Double.class;
+ } else if (clazz == char[].class) {
+ type = Character.class;
+ } else {
+ throw new IndexOutOfBoundsException("Type could not be determined for array");
+ }
+
+ } else if (pos == 0) {
+ type = outTypeInfo.getTypeClass();
+ } else {
+ throw new IndexOutOfBoundsException("Position is out of range");
+ }
+ return type;
+ }
+
+ /**
+ * Checks if the given field position is allowed for the output type
+ *
+ * @param pos
+ * Position to check
+ */
+ protected void checkFieldRange(int pos) {
+ try {
+ getClassAtPos(pos);
+ } catch (IndexOutOfBoundsException e) {
+ throw new RuntimeException("Selected field is out of range");
+
+ }
+ }
+
+ private void validateMerge(String id) {
+ for (DataStream<OUT> ds : this.mergedStreams) {
+ if (ds.getId().equals(id)) {
+ throw new RuntimeException("A DataStream cannot be merged with itself");
+ }
+ }
+ }
+
+ /**
* Creates a copy of the {@link DataStream}
*
* @return The copy
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index dd26e64..622c03d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -30,9 +30,9 @@ import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.A
import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowingInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowingGroupInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowingReduceInvokable;
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
+import org.apache.flink.streaming.api.invokable.operator.WindowGroupReduceInvokable;
+import org.apache.flink.streaming.api.invokable.operator.WindowReduceInvokable;
import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
@@ -57,14 +57,17 @@ public class WindowedDataStream<OUT> {
protected boolean isGrouped;
protected KeySelector<OUT, ?> keySelector;
- protected List<WindowingHelper<OUT>> triggerPolicies;
- protected List<WindowingHelper<OUT>> evictionPolicies;
+ protected List<WindowingHelper<OUT>> triggerHelpers;
+ protected List<WindowingHelper<OUT>> evictionHelpers;
+
+ protected LinkedList<TriggerPolicy<OUT>> userTriggers;
+ protected LinkedList<EvictionPolicy<OUT>> userEvicters;
protected WindowedDataStream(DataStream<OUT> dataStream, WindowingHelper<OUT>... policyHelpers) {
this.dataStream = dataStream.copy();
- this.triggerPolicies = new ArrayList<WindowingHelper<OUT>>();
+ this.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
for (WindowingHelper<OUT> helper : policyHelpers) {
- this.triggerPolicies.add(helper);
+ this.triggerHelpers.add(helper);
}
if (dataStream instanceof GroupedDataStream) {
@@ -76,86 +79,49 @@ public class WindowedDataStream<OUT> {
}
}
- protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
- this.dataStream = windowedDataStream.dataStream.copy();
- this.isGrouped = windowedDataStream.isGrouped;
- this.keySelector = windowedDataStream.keySelector;
- this.triggerPolicies = windowedDataStream.triggerPolicies;
- this.evictionPolicies = windowedDataStream.evictionPolicies;
- }
-
- protected LinkedList<TriggerPolicy<OUT>> getTriggers() {
- LinkedList<TriggerPolicy<OUT>> triggerPolicyList = new LinkedList<TriggerPolicy<OUT>>();
+ protected WindowedDataStream(DataStream<OUT> dataStream, List<TriggerPolicy<OUT>> triggers,
+ List<EvictionPolicy<OUT>> evicters) {
+ this.dataStream = dataStream.copy();
- for (WindowingHelper<OUT> helper : triggerPolicies) {
- triggerPolicyList.add(helper.toTrigger());
+ if (triggers != null) {
+ this.userTriggers = new LinkedList<TriggerPolicy<OUT>>();
+ this.userTriggers.addAll(triggers);
}
- return triggerPolicyList;
- }
+ if (evicters != null) {
+ this.userEvicters = new LinkedList<EvictionPolicy<OUT>>();
+ this.userEvicters.addAll(evicters);
+ }
- protected LinkedList<EvictionPolicy<OUT>> getEvicters() {
- LinkedList<EvictionPolicy<OUT>> evictionPolicyList = new LinkedList<EvictionPolicy<OUT>>();
+ if (dataStream instanceof GroupedDataStream) {
+ this.isGrouped = true;
+ this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
- if (evictionPolicies != null) {
- for (WindowingHelper<OUT> helper : evictionPolicies) {
- evictionPolicyList.add(helper.toEvict());
- }
} else {
- evictionPolicyList.add(new TumblingEvictionPolicy<OUT>());
+ this.isGrouped = false;
}
-
- return evictionPolicyList;
}
- protected LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
- LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
- // Add Time triggers to central triggers
- for (TriggerPolicy<OUT> trigger : getTriggers()) {
- if (trigger instanceof TimeTriggerPolicy) {
- cTriggers.add(trigger);
- }
- }
- return cTriggers;
+ protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
+ this.dataStream = windowedDataStream.dataStream.copy();
+ this.isGrouped = windowedDataStream.isGrouped;
+ this.keySelector = windowedDataStream.keySelector;
+ this.triggerHelpers = windowedDataStream.triggerHelpers;
+ this.evictionHelpers = windowedDataStream.evictionHelpers;
+ this.userTriggers = windowedDataStream.userTriggers;
+ this.userEvicters = windowedDataStream.userEvicters;
}
- protected LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
- LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
-
- // Everything except Time triggers are distributed
- for (TriggerPolicy<OUT> trigger : getTriggers()) {
- if (!(trigger instanceof TimeTriggerPolicy)) {
- dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
- }
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ public WindowedDataStream<OUT> every(WindowingHelper... policyHelpers) {
+ WindowedDataStream<OUT> ret = this.copy();
+ if (ret.evictionHelpers == null) {
+ ret.evictionHelpers = ret.triggerHelpers;
+ ret.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
}
-
- return dTriggers;
- }
-
- protected LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
- LinkedList<CloneableEvictionPolicy<OUT>> evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
-
- for (EvictionPolicy<OUT> evicter : getEvicters()) {
- evicters.add((CloneableEvictionPolicy<OUT>) evicter);
+ for (WindowingHelper<OUT> helper : policyHelpers) {
+ ret.triggerHelpers.add(helper);
}
-
- return evicters;
- }
-
- /**
- * Groups the elements of the {@link WindowedDataStream} by the given
- * {@link KeySelector} to be used with grouped operators.
- *
- * @param keySelector
- * The specification of the key on which the
- * {@link WindowedDataStream} will be grouped.
- * @return The transformed {@link WindowedDataStream}
- */
- public WindowedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
- WindowedDataStream<OUT> ret = this.copy();
- ret.dataStream = ret.dataStream.groupBy(keySelector);
- ret.isGrouped = true;
- ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
return ret;
}
@@ -192,12 +158,29 @@ public class WindowedDataStream<OUT> {
}
/**
+ * Groups the elements of the {@link WindowedDataStream} by the given
+ * {@link KeySelector} to be used with grouped operators.
+ *
+ * @param keySelector
+ * The specification of the key on which the
+ * {@link WindowedDataStream} will be grouped.
+ * @return The transformed {@link WindowedDataStream}
+ */
+ public WindowedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
+ WindowedDataStream<OUT> ret = this.copy();
+ ret.dataStream = ret.dataStream.groupBy(keySelector);
+ ret.isGrouped = true;
+ ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
+ return ret;
+ }
+
+ /**
* This is a prototype implementation for new windowing features based on
* trigger and eviction policies
*
- * @param triggerPolicies
+ * @param triggerHelpers
* A list of trigger policies
- * @param evictionPolicies
+ * @param evictionHelpers
* A list of eviction policies
* @param sample
* A sample of the OUT data type required to gather type
@@ -442,63 +425,123 @@ public class WindowedDataStream<OUT> {
AggregationType.MAXBY, first));
}
- /**
- * Gets the output type.
- *
- * @return The output type.
- */
- public TypeInformation<OUT> getOutputType() {
- return dataStream.getOutputType();
+ private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregator) {
+ StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregator);
+
+ SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.addFunction("windowReduce",
+ aggregator, dataStream.outTypeWrapper, dataStream.outTypeWrapper, invokable);
+
+ return returnStream;
}
- protected WindowedDataStream<OUT> copy() {
- return new WindowedDataStream<OUT>(this);
+ private LinkedList<TriggerPolicy<OUT>> getTriggers() {
+
+ LinkedList<TriggerPolicy<OUT>> triggers = new LinkedList<TriggerPolicy<OUT>>();
+
+ if (triggerHelpers != null) {
+ for (WindowingHelper<OUT> helper : triggerHelpers) {
+ triggers.add(helper.toTrigger());
+ }
+ }
+
+ if (userTriggers != null) {
+ triggers.addAll(userTriggers);
+ }
+
+ return triggers;
+
}
- @SuppressWarnings({ "unchecked", "rawtypes" })
- public WindowedDataStream<OUT> every(WindowingHelper... policyHelpers) {
- WindowedDataStream<OUT> ret = this.copy();
- if (ret.evictionPolicies == null) {
- ret.evictionPolicies = ret.triggerPolicies;
- ret.triggerPolicies = new ArrayList<WindowingHelper<OUT>>();
+ private LinkedList<EvictionPolicy<OUT>> getEvicters() {
+
+ LinkedList<EvictionPolicy<OUT>> evicters = new LinkedList<EvictionPolicy<OUT>>();
+
+ if (evictionHelpers != null) {
+ for (WindowingHelper<OUT> helper : evictionHelpers) {
+ evicters.add(helper.toEvict());
+ }
+ } else {
+ if (userEvicters == null) {
+ evicters.add(new TumblingEvictionPolicy<OUT>());
+ }
}
- for (WindowingHelper<OUT> helper : policyHelpers) {
- ret.triggerPolicies.add(helper);
+
+ if (userEvicters != null) {
+ evicters.addAll(userEvicters);
}
- return ret;
+
+ return evicters;
}
- private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregator) {
- StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregator);
+ private LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
+ LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
+ // Add Time triggers to central triggers
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (trigger instanceof TimeTriggerPolicy) {
+ cTriggers.add(trigger);
+ }
+ }
+ return cTriggers;
+ }
- SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.addFunction("windowReduce",
- aggregator, dataStream.outTypeWrapper, dataStream.outTypeWrapper, invokable);
+ private LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
+ LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
- return returnStream;
+ // Everything except Time triggers are distributed
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (!(trigger instanceof TimeTriggerPolicy)) {
+ dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
+ }
+ }
+
+ return dTriggers;
}
- protected <R> StreamInvokable<OUT, R> getReduceGroupInvokable(
- GroupReduceFunction<OUT, R> reducer) {
+ private LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
+ LinkedList<CloneableEvictionPolicy<OUT>> evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
+
+ for (EvictionPolicy<OUT> evicter : getEvicters()) {
+ evicters.add((CloneableEvictionPolicy<OUT>) evicter);
+ }
+
+ return evicters;
+ }
+
+ private <R> StreamInvokable<OUT, R> getReduceGroupInvokable(GroupReduceFunction<OUT, R> reducer) {
StreamInvokable<OUT, R> invokable;
if (isGrouped) {
- invokable = new GroupedWindowingInvokable<OUT, R>(reducer, keySelector,
+ invokable = new GroupedWindowInvokable<OUT, R>(reducer, keySelector,
getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers());
} else {
- invokable = new WindowingGroupInvokable<OUT, R>(reducer, getTriggers(), getEvicters());
+ invokable = new WindowGroupReduceInvokable<OUT, R>(reducer, getTriggers(),
+ getEvicters());
}
return invokable;
}
- protected StreamInvokable<OUT, OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
+ private StreamInvokable<OUT, OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
StreamInvokable<OUT, OUT> invokable;
if (isGrouped) {
- invokable = new GroupedWindowingInvokable<OUT, OUT>(reducer, keySelector,
+ invokable = new GroupedWindowInvokable<OUT, OUT>(reducer, keySelector,
getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers());
} else {
- invokable = new WindowingReduceInvokable<OUT>(reducer, getTriggers(), getEvicters());
+ invokable = new WindowReduceInvokable<OUT>(reducer, getTriggers(), getEvicters());
}
return invokable;
}
+
+ /**
+ * Gets the output type.
+ *
+ * @return The output type.
+ */
+ public TypeInformation<OUT> getOutputType() {
+ return dataStream.getOutputType();
+ }
+
+ protected WindowedDataStream<OUT> copy() {
+ return new WindowedDataStream<OUT>(this);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
new file mode 100644
index 0000000..844a488
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This invokable allows windowing based on {@link TriggerPolicy} and
+ * {@link EvictionPolicy} instances including their active and cloneable
+ * versions. It is additionally aware of the creation of windows per group.
+ *
+ * A {@link KeySelector} is used to specify the key position or key extraction.
+ * The {@link ReduceFunction} will be executed on each group separately. Trigger
+ * policies might either be centralized or distributed. Eviction policies are
+ * always distributed. A distributed policy have to be a
+ * {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it will
+ * be cloned to have separated instances for each group. At the startup time the
+ * distributed policies will be stored as sample, and only clones of them will
+ * be used to maintain the groups. Therefore, each group starts with the initial
+ * policy states.
+ *
+ * While a distributed policy only gets notified with the elements belonging to
+ * the respective group, a centralized policy get notified with all arriving
+ * elements. When a centralized trigger occurred, all groups get triggered. This
+ * is done by submitting the element which caused the trigger as real element to
+ * the groups it belongs to and as fake element to all other groups. Within the
+ * groups the element might be further processed, causing more triggered,
+ * prenotifications of active distributed policies and evictions like usual.
+ *
+ * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+ * implement the
+ * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+ * method. Fake elements created on prenotification will be forwarded to all
+ * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
+ * it forwards/distributed calls all groups.
+ *
+ * @param <IN>
+ * The type of input elements handled by this operator invokable.
+ */
+public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+
+ /**
+ * Auto-generated serial version UID
+ */
+ private static final long serialVersionUID = -3469545957144404137L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(GroupedWindowInvokable.class);
+
+ private KeySelector<IN, ?> keySelector;
+ private Configuration parameters;
+ private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+ private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
+ private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
+ private Map<Object, WindowInvokable<IN, OUT>> windowingGroups = new HashMap<Object, WindowInvokable<IN, OUT>>();
+ private LinkedList<Thread> activePolicyThreads = new LinkedList<Thread>();
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+
+ /**
+ * This constructor creates an instance of the grouped windowing invokable.
+ * A {@link KeySelector} is used to specify the key position or key
+ * extraction. The {@link ReduceFunction} will be executed on each group
+ * separately. Trigger policies might either be centralized or distributed.
+ * Eviction policies are always distributed. A distributed policy have to be
+ * a {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it
+ * will be cloned to have separated instances for each group. At the startup
+ * time the distributed policies will be stored as sample, and only clones
+ * of them will be used to maintain the groups. Therefore, each group starts
+ * with the initial policy states.
+ *
+ * While a distributed policy only gets notified with the elements belonging
+ * to the respective group, a centralized policy get notified with all
+ * arriving elements. When a centralized trigger occurred, all groups get
+ * triggered. This is done by submitting the element which caused the
+ * trigger as real element to the groups it belongs to and as fake element
+ * to all other groups. Within the groups the element might be further
+ * processed, causing more triggered, prenotifications of active distributed
+ * policies and evictions like usual.
+ *
+ * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+ * implement the
+ * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+ * method. Fake elements created on prenotification will be forwarded to all
+ * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
+ * that it forwards/distributed calls all groups.
+ *
+ * @param userFunction
+ * The user defined function.
+ * @param keySelector
+ * A key selector to extract the key for the groups from the
+ * input data.
+ * @param distributedTriggerPolicies
+ * Trigger policies to be distributed and maintained individually
+ * within each group.
+ * @param distributedEvictionPolicies
+ * Eviction policies to be distributed and maintained
+ * individually within each group. There are no central eviction
+ * policies because there is no central element buffer but only a
+ * buffer per group. Therefore evictions might always be done per
+ * group.
+ * @param centralTriggerPolicies
+ * Trigger policies which will only exist once at a central
+ * place. In case a central policy triggers, it will cause all
+ * groups to be emitted. (Remark: Empty groups cannot be emitted.
+ * If only one element is contained a group, this element itself
+ * is returned as aggregated result.)
+ */
+ public GroupedWindowInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
+ LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
+ LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
+ LinkedList<TriggerPolicy<IN>> centralTriggerPolicies) {
+
+ super(userFunction);
+ this.keySelector = keySelector;
+ this.centralTriggerPolicies = centralTriggerPolicies;
+ this.distributedTriggerPolicies = distributedTriggerPolicies;
+ this.distributedEvictionPolicies = distributedEvictionPolicies;
+
+ for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
+ if (trigger instanceof ActiveTriggerPolicy) {
+ this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
+ }
+ }
+ }
+
+ @Override
+ protected void immutableInvoke() throws Exception {
+ // Prevent empty data streams
+ if ((reuse = recordIterator.next(reuse)) == null) {
+ throw new RuntimeException("DataStream must not be empty");
+ }
+
+ // Continuously run
+ while (reuse != null) {
+ WindowInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector
+ .getKey(reuse.getObject()));
+ if (groupInvokable == null) {
+ groupInvokable = makeNewGroup(reuse);
+ }
+
+ // Run the precalls for central active triggers
+ for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
+ Object[] result = trigger.preNotifyTrigger(reuse.getObject());
+ for (Object in : result) {
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ group.processFakeElement(in, trigger);
+ }
+ }
+ }
+
+ // Process non-active central triggers
+ for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
+ if (triggerPolicy.notifyTrigger(reuse.getObject())) {
+ currentTriggerPolicies.add(triggerPolicy);
+ }
+ }
+
+ if (currentTriggerPolicies.isEmpty()) {
+ // only add the element to its group
+ groupInvokable.processRealElement(reuse.getObject());
+ } else {
+ // call user function for all groups
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ if (group == groupInvokable) {
+ // process real with initialized policies
+ group.processRealElement(reuse.getObject(), currentTriggerPolicies);
+ } else {
+ // process like a fake but also initialized with
+ // policies
+ group.externalTriggerFakeElement(reuse.getObject(), currentTriggerPolicies);
+ }
+ }
+ }
+
+ // clear current trigger list
+ currentTriggerPolicies.clear();
+
+ // Recreate the reuse-StremRecord object and load next StreamRecord
+ resetReuse();
+ reuse = recordIterator.next(reuse);
+ }
+
+ // Stop all remaining threads from policies
+ for (Thread t : activePolicyThreads) {
+ t.interrupt();
+ }
+
+ // finally trigger the buffer.
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ group.emitFinalWindow(centralTriggerPolicies);
+ }
+
+ }
+
+ /**
+ * This method creates a new group. The method gets called in case an
+ * element arrives which has a key which was not seen before. The method
+ * created a nested {@link WindowInvokable} and therefore created clones
+ * of all distributed trigger and eviction policies.
+ *
+ * @param element
+ * The element which leads to the generation of a new group
+ * (previously unseen key)
+ * @throws Exception
+ * In case the {@link KeySelector} throws an exception in
+ * {@link KeySelector#getKey(Object)}, the exception is not
+ * catched by this method.
+ */
+ @SuppressWarnings("unchecked")
+ private WindowInvokable<IN, OUT> makeNewGroup(StreamRecord<IN> element) throws Exception {
+ // clone the policies
+ LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
+ for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
+ clonedDistributedTriggerPolicies.add(trigger.clone());
+ }
+ for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
+ clonedDistributedEvictionPolicies.add(eviction.clone());
+ }
+
+ WindowInvokable<IN, OUT> groupInvokable;
+ if (userFunction instanceof ReduceFunction) {
+ groupInvokable = (WindowInvokable<IN, OUT>) new WindowReduceInvokable<IN>(
+ (ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
+ clonedDistributedEvictionPolicies);
+ } else {
+ groupInvokable = new WindowGroupReduceInvokable<IN, OUT>(
+ (GroupReduceFunction<IN, OUT>) userFunction, clonedDistributedTriggerPolicies,
+ clonedDistributedEvictionPolicies);
+ }
+
+ groupInvokable.initialize(collector, recordIterator, inSerializer, isMutable);
+ groupInvokable.open(this.parameters);
+ windowingGroups.put(keySelector.getKey(element.getObject()), groupInvokable);
+
+ return groupInvokable;
+ }
+
+ @Override
+ protected void mutableInvoke() throws Exception {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("There is currently no mutable implementation of this operator. Immutable version is used.");
+ }
+ immutableInvoke();
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ // This method gets never called directly. The user function calls are
+ // all delegated to the invokable instanced which handle/represent the
+ // groups.
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ this.parameters = parameters;
+ for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
+ Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+ if (target != null) {
+ Thread thread = new Thread(target);
+ activePolicyThreads.add(thread);
+ thread.start();
+ }
+ }
+ };
+
+ /**
+ * This callback class allows to handle the the callbacks done by threads
+ * defined in active trigger policies
+ *
+ * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
+ */
+ private class WindowingCallback implements ActiveTriggerCallback {
+ private ActiveTriggerPolicy<IN> policy;
+
+ public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+ this.policy = policy;
+ }
+
+ @Override
+ public void sendFakeElement(Object datapoint) {
+ for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
+ group.processFakeElement(datapoint, policy);
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
deleted file mode 100644
index c341e56..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This invokable allows windowing based on {@link TriggerPolicy} and
- * {@link EvictionPolicy} instances including their active and cloneable
- * versions. It is additionally aware of the creation of windows per group.
- *
- * A {@link KeySelector} is used to specify the key position or key extraction.
- * The {@link ReduceFunction} will be executed on each group separately. Trigger
- * policies might either be centralized or distributed. Eviction policies are
- * always distributed. A distributed policy have to be a
- * {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it will
- * be cloned to have separated instances for each group. At the startup time the
- * distributed policies will be stored as sample, and only clones of them will
- * be used to maintain the groups. Therefore, each group starts with the initial
- * policy states.
- *
- * While a distributed policy only gets notified with the elements belonging to
- * the respective group, a centralized policy get notified with all arriving
- * elements. When a centralized trigger occurred, all groups get triggered. This
- * is done by submitting the element which caused the trigger as real element to
- * the groups it belongs to and as fake element to all other groups. Within the
- * groups the element might be further processed, causing more triggered,
- * prenotifications of active distributed policies and evictions like usual.
- *
- * Central policies can be instance of {@link ActiveTriggerPolicy} and also
- * implement the
- * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
- * method. Fake elements created on prenotification will be forwarded to all
- * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
- * it forwards/distributed calls all groups.
- *
- * @param <IN>
- * The type of input elements handled by this operator invokable.
- */
-public class GroupedWindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
-
- /**
- * Auto-generated serial version UID
- */
- private static final long serialVersionUID = -3469545957144404137L;
-
- private static final Logger LOG = LoggerFactory.getLogger(GroupedWindowingInvokable.class);
-
- private KeySelector<IN, ?> keySelector;
- private Configuration parameters;
- private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
- private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
- private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
- private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
- private Map<Object, WindowingInvokable<IN, OUT>> windowingGroups = new HashMap<Object, WindowingInvokable<IN, OUT>>();
- private LinkedList<Thread> activePolicyThreads = new LinkedList<Thread>();
- private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-
- /**
- * This constructor creates an instance of the grouped windowing invokable.
- * A {@link KeySelector} is used to specify the key position or key
- * extraction. The {@link ReduceFunction} will be executed on each group
- * separately. Trigger policies might either be centralized or distributed.
- * Eviction policies are always distributed. A distributed policy have to be
- * a {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it
- * will be cloned to have separated instances for each group. At the startup
- * time the distributed policies will be stored as sample, and only clones
- * of them will be used to maintain the groups. Therefore, each group starts
- * with the initial policy states.
- *
- * While a distributed policy only gets notified with the elements belonging
- * to the respective group, a centralized policy get notified with all
- * arriving elements. When a centralized trigger occurred, all groups get
- * triggered. This is done by submitting the element which caused the
- * trigger as real element to the groups it belongs to and as fake element
- * to all other groups. Within the groups the element might be further
- * processed, causing more triggered, prenotifications of active distributed
- * policies and evictions like usual.
- *
- * Central policies can be instance of {@link ActiveTriggerPolicy} and also
- * implement the
- * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
- * method. Fake elements created on prenotification will be forwarded to all
- * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
- * that it forwards/distributed calls all groups.
- *
- * @param userFunction
- * The user defined function.
- * @param keySelector
- * A key selector to extract the key for the groups from the
- * input data.
- * @param distributedTriggerPolicies
- * Trigger policies to be distributed and maintained individually
- * within each group.
- * @param distributedEvictionPolicies
- * Eviction policies to be distributed and maintained
- * individually within each group. There are no central eviction
- * policies because there is no central element buffer but only a
- * buffer per group. Therefore evictions might always be done per
- * group.
- * @param centralTriggerPolicies
- * Trigger policies which will only exist once at a central
- * place. In case a central policy triggers, it will cause all
- * groups to be emitted. (Remark: Empty groups cannot be emitted.
- * If only one element is contained a group, this element itself
- * is returned as aggregated result.)
- */
- public GroupedWindowingInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
- LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
- LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
- LinkedList<TriggerPolicy<IN>> centralTriggerPolicies) {
-
- super(userFunction);
- this.keySelector = keySelector;
- this.centralTriggerPolicies = centralTriggerPolicies;
- this.distributedTriggerPolicies = distributedTriggerPolicies;
- this.distributedEvictionPolicies = distributedEvictionPolicies;
-
- for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
- if (trigger instanceof ActiveTriggerPolicy) {
- this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
- }
- }
- }
-
- @Override
- protected void immutableInvoke() throws Exception {
- // Prevent empty data streams
- if ((reuse = recordIterator.next(reuse)) == null) {
- throw new RuntimeException("DataStream must not be empty");
- }
-
- // Continuously run
- while (reuse != null) {
- WindowingInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector
- .getKey(reuse.getObject()));
- if (groupInvokable == null) {
- groupInvokable = makeNewGroup(reuse);
- }
-
- // Run the precalls for central active triggers
- for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
- Object[] result = trigger.preNotifyTrigger(reuse.getObject());
- for (Object in : result) {
- for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
- group.processFakeElement(in, trigger);
- }
- }
- }
-
- // Process non-active central triggers
- for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
- if (triggerPolicy.notifyTrigger(reuse.getObject())) {
- currentTriggerPolicies.add(triggerPolicy);
- }
- }
-
- if (currentTriggerPolicies.isEmpty()) {
- // only add the element to its group
- groupInvokable.processRealElement(reuse.getObject());
- } else {
- // call user function for all groups
- for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
- if (group == groupInvokable) {
- // process real with initialized policies
- group.processRealElement(reuse.getObject(), currentTriggerPolicies);
- } else {
- // process like a fake but also initialized with
- // policies
- group.externalTriggerFakeElement(reuse.getObject(), currentTriggerPolicies);
- }
- }
- }
-
- // clear current trigger list
- currentTriggerPolicies.clear();
-
- // Recreate the reuse-StremRecord object and load next StreamRecord
- resetReuse();
- reuse = recordIterator.next(reuse);
- }
-
- // Stop all remaining threads from policies
- for (Thread t : activePolicyThreads) {
- t.interrupt();
- }
-
- // finally trigger the buffer.
- for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
- group.emitFinalWindow(centralTriggerPolicies);
- }
-
- }
-
- /**
- * This method creates a new group. The method gets called in case an
- * element arrives which has a key which was not seen before. The method
- * created a nested {@link WindowingInvokable} and therefore created clones
- * of all distributed trigger and eviction policies.
- *
- * @param element
- * The element which leads to the generation of a new group
- * (previously unseen key)
- * @throws Exception
- * In case the {@link KeySelector} throws an exception in
- * {@link KeySelector#getKey(Object)}, the exception is not
- * catched by this method.
- */
- @SuppressWarnings("unchecked")
- private WindowingInvokable<IN, OUT> makeNewGroup(StreamRecord<IN> element) throws Exception {
- // clone the policies
- LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
- LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
- for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
- clonedDistributedTriggerPolicies.add(trigger.clone());
- }
- for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
- clonedDistributedEvictionPolicies.add(eviction.clone());
- }
-
- WindowingInvokable<IN, OUT> groupInvokable;
- if (userFunction instanceof ReduceFunction) {
- groupInvokable = (WindowingInvokable<IN, OUT>) new WindowingReduceInvokable<IN>(
- (ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
- clonedDistributedEvictionPolicies);
- } else {
- groupInvokable = new WindowingGroupInvokable<IN, OUT>(
- (GroupReduceFunction<IN, OUT>) userFunction, clonedDistributedTriggerPolicies,
- clonedDistributedEvictionPolicies);
- }
-
- groupInvokable.initialize(collector, recordIterator, inSerializer, isMutable);
- groupInvokable.open(this.parameters);
- windowingGroups.put(keySelector.getKey(element.getObject()), groupInvokable);
-
- return groupInvokable;
- }
-
- @Override
- protected void mutableInvoke() throws Exception {
- if (LOG.isInfoEnabled()) {
- LOG.info("There is currently no mutable implementation of this operator. Immutable version is used.");
- }
- immutableInvoke();
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- // This method gets never called directly. The user function calls are
- // all delegated to the invokable instanced which handle/represent the
- // groups.
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- super.open(parameters);
- this.parameters = parameters;
- for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
- Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
- if (target != null) {
- Thread thread = new Thread(target);
- activePolicyThreads.add(thread);
- thread.start();
- }
- }
- };
-
- /**
- * This callback class allows to handle the the callbacks done by threads
- * defined in active trigger policies
- *
- * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
- */
- private class WindowingCallback implements ActiveTriggerCallback {
- private ActiveTriggerPolicy<IN> policy;
-
- public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
- this.policy = policy;
- }
-
- @Override
- public void sendFakeElement(Object datapoint) {
- for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
- group.processFakeElement(datapoint, policy);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
new file mode 100644
index 0000000..9d0b584
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.LinkedList;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public class WindowGroupReduceInvokable<IN, OUT> extends WindowInvokable<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+ GroupReduceFunction<IN, OUT> reducer;
+
+ public WindowGroupReduceInvokable(GroupReduceFunction<IN, OUT> userFunction,
+ LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction, triggerPolicies, evictionPolicies);
+ this.reducer = userFunction;
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ reducer.reduce(buffer, collector);
+ }
+
+}
[08/34] incubator-flink git commit: [streaming] Added extractor
implementations for various extractions and conversions. (incl. test cases)
Posted by mb...@apache.org.
[streaming] Added extractor implementations for various extractions and conversions. (incl. test cases)
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/05c15198
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/05c15198
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/05c15198
Branch: refs/heads/master
Commit: 05c15198c6cb3edaed206c67727e7a4d08ad216b
Parents: 6f3bf57
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:40:22 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../api/windowing/extractor/ArrayFromTuple.java | 74 ++++++++++++
.../extractor/ConcatinatedExtract.java | 68 +++++++++++
.../api/windowing/extractor/FieldFromArray.java | 59 ++++++++++
.../api/windowing/extractor/FieldFromTuple.java | 58 +++++++++
.../windowing/extractor/FieldsFromArray.java | 67 +++++++++++
.../windowing/extractor/FieldsFromTuple.java | 53 +++++++++
.../windowing/extractor/ArrayFromTupleTest.java | 118 +++++++++++++++++++
.../extractor/ConcatinatedExtractTest.java | 83 +++++++++++++
.../windowing/extractor/FieldFromArrayTest.java | 55 +++++++++
.../windowing/extractor/FieldFromTupleTest.java | 84 +++++++++++++
.../extractor/FieldsFromArrayTest.java | 108 +++++++++++++++++
.../extractor/FieldsFromTupleTest.java | 106 +++++++++++++++++
12 files changed, 933 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
new file mode 100644
index 0000000..ee878ac
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTuple.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import org.apache.flink.api.java.tuple.Tuple;
+
+/**
+ * Converts a Tuple to an Object-Array. The field which should be included in
+ * the array can selected and reordered as needed.
+ */
+public class ArrayFromTuple implements Extractor<Tuple, Object[]> {
+
+ /**
+ * Auto generated version id
+ */
+ private static final long serialVersionUID = -6076121226427616818L;
+ int[] order = null;
+
+ /**
+ * Using this constructor the extractor will convert the whole tuple (all
+ * fields in the original order) to an array.
+ */
+ public ArrayFromTuple() {
+ // noting to do
+ }
+
+ /**
+ * Using this constructor the extractor will combine the fields as specified
+ * in the indexes parameter in an object array.
+ *
+ * @param indexes
+ * the field ids (enumerated from 0)
+ */
+ public ArrayFromTuple(int... indexes) {
+ this.order = indexes;
+ }
+
+ @Override
+ public Object[] extract(Tuple in) {
+ Object[] output;
+
+ if (order == null) {
+ // copy the whole tuple
+ output = new Object[in.getArity()];
+ for (int i = 0; i < in.getArity(); i++) {
+ output[i] = in.getField(i);
+ }
+ } else {
+ // copy user specified order
+ output = new Object[order.length];
+ for (int i = 0; i < order.length; i++) {
+ output[i] = in.getField(order[i]);
+ }
+ }
+
+ return output;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
new file mode 100644
index 0000000..9048a3c
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtract.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+/**
+ * Combines two extractors which will be executed one after each other.
+ *
+ * @param <FROM>
+ * The input type of the first extractor.
+ * @param <OVER>
+ * The output type of the first and the input type of the second
+ * extractor.
+ * @param <TO>
+ * The output type of the second extractor and the output type of the
+ * over all extraction.
+ */
+public class ConcatinatedExtract<FROM, OVER, TO> implements Extractor<FROM, TO> {
+
+ /**
+ * auto-generated id
+ */
+ private static final long serialVersionUID = -7807197760725651752L;
+
+ private Extractor<FROM, OVER> e1;
+ private Extractor<OVER, TO> e2;
+
+ /**
+ * Combines two extractors which will be executed one after each other.
+ *
+ * @param e1
+ * First extractor: This extractor gets applied to the input data
+ * first. Its output as then passed as input to the second
+ * extractor.
+ * @param e2
+ * Second extractor: This extractor gets the output of the first
+ * extractor as input. Its output is then the result of the over
+ * all extraction.
+ */
+ public ConcatinatedExtract(Extractor<FROM, OVER> e1, Extractor<OVER, TO> e2) {
+ this.e1 = e1;
+ this.e2 = e2;
+ }
+
+ @Override
+ public TO extract(FROM in) {
+ return e2.extract(e1.extract(in));
+ }
+
+ public <OUT> ConcatinatedExtract<FROM, TO, OUT> add(Extractor<TO, OUT> e3) {
+ return new ConcatinatedExtract<FROM, TO, OUT>(this, e3);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
new file mode 100644
index 0000000..0568276
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArray.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import java.lang.reflect.Array;
+
+/**
+ * Extracts a single field out of an array.
+ *
+ * @param <OUT>
+ * The type of the extracted field.
+ */
+public class FieldFromArray<OUT> implements Extractor<Object, OUT> {
+
+ /**
+ * Auto-gernated version id
+ */
+ private static final long serialVersionUID = -5161386546695574359L;
+ private int fieldId = 0;
+
+ /**
+ * Extracts the first field (id 0) from the array
+ */
+ public FieldFromArray() {
+ // noting to do => will use default 0
+ }
+
+ /**
+ * Extracts the field with the given id from the array.
+ *
+ * @param fieldId
+ * The id of the field which will be extracted from the array.
+ */
+ public FieldFromArray(int fieldId) {
+ this.fieldId = fieldId;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public OUT extract(Object in) {
+ return (OUT) Array.get(in, fieldId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
new file mode 100644
index 0000000..07b38f5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTuple.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import org.apache.flink.api.java.tuple.Tuple;
+
+/**
+ * Extracts a single field out of a tuple.
+ *
+ * @param <OUT>
+ * The type of the extracted field.
+ */
+public class FieldFromTuple<OUT> implements Extractor<Tuple, OUT> {
+
+ /**
+ * Auto-gernated version id
+ */
+ private static final long serialVersionUID = -5161386546695574359L;
+ private int fieldId = 0;
+
+ /**
+ * Extracts the first field (id 0) from the tuple
+ */
+ public FieldFromTuple() {
+ // noting to do => will use default 0
+ }
+
+ /**
+ * Extracts the field with the given id from the tuple.
+ *
+ * @param fieldId
+ * The id of the field which will be extracted from the tuple.
+ */
+ public FieldFromTuple(int fieldId) {
+ this.fieldId = fieldId;
+ }
+
+ @Override
+ public OUT extract(Tuple in) {
+ return in.getField(fieldId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
new file mode 100644
index 0000000..4e98689
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArray.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import java.lang.reflect.Array;
+
+/**
+ * Extracts multiple fields from an array and puts them into a new array of the
+ * specified type.
+ *
+ * @param <OUT>
+ * The type of the output array. If out is set to String, the output
+ * of the extractor will be a String[]. If it is set to String[] the
+ * output will be String[][].
+ */
+public class FieldsFromArray<OUT> implements Extractor<Object, OUT[]> {
+
+ /**
+ * Auto-generated version id
+ */
+ private static final long serialVersionUID = 8075055384516397670L;
+ private int[] order;
+ private Class<OUT> clazz;
+
+ /**
+ * Extracts multiple fields from an array and puts them in the given order
+ * into a new array of the specified type.
+ *
+ * @param clazz
+ * the Class object representing the component type of the new
+ * array
+ * @param indexes
+ * The indexes of the fields to be extracted. Any order is
+ * possible, but not more than 255 fields due to limitations in
+ * {@link Array#newInstance(Class, int...)}.
+ */
+ public FieldsFromArray(Class<OUT> clazz, int... indexes) {
+ this.order = indexes;
+ this.clazz = clazz;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public OUT[] extract(Object in) {
+ OUT[] output = (OUT[]) Array.newInstance(clazz, order.length);
+ for (int i = 0; i < order.length; i++) {
+ output[i] = (OUT) Array.get(in, this.order[i]);
+ }
+ return output;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
new file mode 100644
index 0000000..58bb4e0
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTuple.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import org.apache.flink.api.java.tuple.Tuple;
+
+/**
+ * Extracts one or more fields of the the type Double from a tuple and puts them
+ * into a new double[]
+ */
+public class FieldsFromTuple implements Extractor<Tuple, double[]> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -2554079091050273761L;
+ int[] indexes;
+
+ /**
+ * Extracts one or more fields of the the type Double from a tuple and puts
+ * them into a new double[] (in the specified order).
+ *
+ * @param indexes
+ * The indexes of the fields to be extracted.
+ */
+ public FieldsFromTuple(int... indexes) {
+ this.indexes = indexes;
+ }
+
+ @Override
+ public double[] extract(Tuple in) {
+ double[] out = new double[indexes.length];
+ for (int i = 0; i < indexes.length; i++) {
+ out[i] = in.getField(indexes[i]);
+ }
+ return out;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
new file mode 100644
index 0000000..17d3974
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ArrayFromTupleTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+import org.apache.flink.streaming.api.windowing.extractor.ArrayFromTuple;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ArrayFromTupleTest {
+
+ private String[] testStrings;
+
+ @Before
+ public void init() {
+ testStrings = new String[Tuple.MAX_ARITY];
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ testStrings[i] = Integer.toString(i);
+ }
+ }
+
+ @Test
+ public void testConvertFromTupleToArray() throws InstantiationException, IllegalAccessException {
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ Tuple currentTuple = (Tuple) CLASSES[i].newInstance();
+ String[] currentArray = new String[i + 1];
+ for (int j = 0; j <= i; j++) {
+ currentTuple.setField(testStrings[j], j);
+ currentArray[j] = testStrings[j];
+ }
+ arrayEqualityCheck(currentArray, new ArrayFromTuple().extract(currentTuple));
+ }
+ }
+
+ @Test
+ public void testUserSpecifiedOrder() throws InstantiationException, IllegalAccessException {
+ Tuple currentTuple = (Tuple) CLASSES[Tuple.MAX_ARITY - 1].newInstance();
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ currentTuple.setField(testStrings[i], i);
+ }
+
+ String[] expected = { testStrings[5], testStrings[3], testStrings[6], testStrings[7],
+ testStrings[0] };
+ arrayEqualityCheck(expected, new ArrayFromTuple(5, 3, 6, 7, 0).extract(currentTuple));
+
+ String[] expected2 = { testStrings[0], testStrings[Tuple.MAX_ARITY - 1] };
+ arrayEqualityCheck(expected2,
+ new ArrayFromTuple(0, Tuple.MAX_ARITY - 1).extract(currentTuple));
+
+ String[] expected3 = { testStrings[Tuple.MAX_ARITY - 1], testStrings[0] };
+ arrayEqualityCheck(expected3,
+ new ArrayFromTuple(Tuple.MAX_ARITY - 1, 0).extract(currentTuple));
+
+ String[] expected4 = { testStrings[13], testStrings[4], testStrings[5], testStrings[4],
+ testStrings[2], testStrings[8], testStrings[6], testStrings[2], testStrings[8],
+ testStrings[3], testStrings[5], testStrings[2], testStrings[16], testStrings[4],
+ testStrings[3], testStrings[2], testStrings[6], testStrings[4], testStrings[7],
+ testStrings[4], testStrings[2], testStrings[8], testStrings[7], testStrings[2] };
+ arrayEqualityCheck(expected4, new ArrayFromTuple(13, 4, 5, 4, 2, 8, 6, 2, 8, 3, 5, 2, 16,
+ 4, 3, 2, 6, 4, 7, 4, 2, 8, 7, 2).extract(currentTuple));
+ }
+
+ private void arrayEqualityCheck(Object[] array1, Object[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
+ }
+ }
+
+ private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
+ Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
+ Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
+ Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
+ Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
+ Tuple24.class, Tuple25.class };
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
new file mode 100644
index 0000000..82a876a
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/ConcatinatedExtractTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.extractor.ArrayFromTuple;
+import org.apache.flink.streaming.api.windowing.extractor.ConcatinatedExtract;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromArray;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromArray;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ConcatinatedExtractTest {
+
+ private String[] testStringArray1 = { "1", "2", "3" };
+ private int[] testIntArray1 = { 1, 2, 3 };
+ private String[] testStringArray2 = { "4", "5", "6" };
+ private int[] testIntArray2 = { 4, 5, 6 };
+ private String[] testStringArray3 = { "7", "8", "9" };
+ private int[] testIntArray3 = { 7, 8, 9 };
+ private Tuple2<String[], int[]>[] testTuple2Array;
+ private Tuple2<String[], int[]> testTuple2;
+ private Tuple2<Tuple2<String[], int[]>, Tuple2<String[], int[]>[]> testData;
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void setupData() {
+ testTuple2Array = new Tuple2[2];
+ testTuple2Array[0] = new Tuple2<String[], int[]>(testStringArray1, testIntArray2);
+ testTuple2Array[1] = new Tuple2<String[], int[]>(testStringArray2, testIntArray1);
+
+ testTuple2 = new Tuple2<String[], int[]>(testStringArray3, testIntArray3);
+
+ testData = new Tuple2<Tuple2<String[], int[]>, Tuple2<String[], int[]>[]>(testTuple2,
+ testTuple2Array);
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void test1() {
+ Extractor ext = new ConcatinatedExtract(new FieldFromTuple(0), new FieldFromTuple(1))
+ .add(new FieldsFromArray(Integer.class, 2, 1, 0));
+ int[] expected = { testIntArray3[2], testIntArray3[1], testIntArray3[0] };
+ assertEquals(new Integer(expected[0]), ((Integer[]) ext.extract(testData))[0]);
+ assertEquals(new Integer(expected[1]), ((Integer[]) ext.extract(testData))[1]);
+ assertEquals(new Integer(expected[2]), ((Integer[]) ext.extract(testData))[2]);
+ }
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void test2() {
+ Extractor ext = new ConcatinatedExtract(new FieldFromTuple(1), // Tuple2<String[],int[]>[]
+ new FieldsFromArray(Tuple2.class, 1)) // Tuple2<String[],int[]>[]
+ .add(new FieldFromArray(0)) // Tuple2<String[],int[]>
+ .add(new ArrayFromTuple(0)) // Object[] (Containing String[])
+ .add(new FieldFromArray(0)) // String[]
+ .add(new FieldFromArray(1)); // String
+
+ String expected2 = testStringArray2[1];
+ assertEquals(expected2, ext.extract(testData));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
new file mode 100644
index 0000000..2d4dbcf
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromArrayTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromArray;
+import org.junit.Test;
+
+public class FieldFromArrayTest {
+
+ String[] testStringArray = { "0", "1", "2", "3", "4" };
+ Integer[] testIntegerArray = { 10, 11, 12, 13, 14 };
+ int[] testIntArray = { 20, 21, 22, 23, 24 };
+
+ @Test
+ public void testStringArray() {
+ for (int i = 0; i < this.testStringArray.length; i++) {
+ assertEquals(this.testStringArray[i],
+ new FieldFromArray<String>(i).extract(testStringArray));
+ }
+ }
+
+ @Test
+ public void testIntegerArray() {
+ for (int i = 0; i < this.testIntegerArray.length; i++) {
+ assertEquals(this.testIntegerArray[i],
+ new FieldFromArray<String>(i).extract(testIntegerArray));
+ }
+ }
+
+ @Test
+ public void testIntArray() {
+ for (int i = 0; i < this.testIntArray.length; i++) {
+ assertEquals(new Integer(this.testIntArray[i]),
+ new FieldFromArray<Integer>(i).extract(testIntArray));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
new file mode 100644
index 0000000..528611a
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldFromTupleTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FieldFromTupleTest {
+
+ private String[] testStrings;
+
+ @Before
+ public void init() {
+ testStrings = new String[Tuple.MAX_ARITY];
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ testStrings[i] = Integer.toString(i);
+ }
+ }
+
+ @Test
+ public void testSingleFieldExtraction() throws InstantiationException, IllegalAccessException {
+ // extract single fields
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ Tuple current = (Tuple) CLASSES[i].newInstance();
+ for (int j = 0; j < i; j++) {
+ current.setField(testStrings[j], j);
+ }
+ for (int j = 0; j < i; j++) {
+ assertEquals(testStrings[j], new FieldFromTuple<String>(j).extract(current));
+ }
+ }
+ }
+
+ private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
+ Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
+ Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
+ Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
+ Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
+ Tuple24.class, Tuple25.class };
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
new file mode 100644
index 0000000..3139aa5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromArrayTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromArray;
+import org.junit.Test;
+
+public class FieldsFromArrayTest {
+
+ String[] testStringArray = { "0", "1", "2", "3", "4" };
+ Integer[] testIntegerArray = { 10, 11, 12, 13, 14 };
+ int[] testIntArray = { 20, 21, 22, 23, 24 };
+
+ @Test
+ public void testStringArray() {
+ // check single field extraction
+ for (int i = 0; i < testStringArray.length; i++) {
+ String[] tmp = { testStringArray[i] };
+ arrayEqualityCheck(tmp,
+ new FieldsFromArray<String>(String.class, i).extract(testStringArray));
+ }
+
+ // check reverse order
+ String[] reverseOrder = new String[testStringArray.length];
+ for (int i = 0; i < testStringArray.length; i++) {
+ reverseOrder[i] = testStringArray[testStringArray.length - i - 1];
+ }
+ arrayEqualityCheck(reverseOrder,
+ new FieldsFromArray<String>(String.class, 4, 3, 2, 1, 0).extract(testStringArray));
+
+ // check picking fields and reorder
+ String[] crazyOrder = { testStringArray[4], testStringArray[1], testStringArray[2] };
+ arrayEqualityCheck(crazyOrder,
+ new FieldsFromArray<String>(String.class, 4, 1, 2).extract(testStringArray));
+ }
+
+ @Test
+ public void testIntegerArray() {
+ // check single field extraction
+ for (int i = 0; i < testIntegerArray.length; i++) {
+ Integer[] tmp = { testIntegerArray[i] };
+ arrayEqualityCheck(tmp,
+ new FieldsFromArray<Integer>(Integer.class, i).extract(testIntegerArray));
+ }
+
+ // check reverse order
+ Integer[] reverseOrder = new Integer[testIntegerArray.length];
+ for (int i = 0; i < testIntegerArray.length; i++) {
+ reverseOrder[i] = testIntegerArray[testIntegerArray.length - i - 1];
+ }
+ arrayEqualityCheck(reverseOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 3, 2, 1, 0)
+ .extract(testIntegerArray));
+
+ // check picking fields and reorder
+ Integer[] crazyOrder = { testIntegerArray[4], testIntegerArray[1], testIntegerArray[2] };
+ arrayEqualityCheck(crazyOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 1, 2).extract(testIntegerArray));
+
+ }
+
+ @Test
+ public void testIntArray() {
+ for (int i = 0; i < testIntArray.length; i++) {
+ Integer[] tmp = { testIntArray[i] };
+ arrayEqualityCheck(tmp,
+ new FieldsFromArray<Integer>(Integer.class, i).extract(testIntArray));
+ }
+
+ // check reverse order
+ Integer[] reverseOrder = new Integer[testIntArray.length];
+ for (int i = 0; i < testIntArray.length; i++) {
+ reverseOrder[i] = testIntArray[testIntArray.length - i - 1];
+ }
+ arrayEqualityCheck(reverseOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 3, 2, 1, 0).extract(testIntArray));
+
+ // check picking fields and reorder
+ Integer[] crazyOrder = { testIntArray[4], testIntArray[1], testIntArray[2] };
+ arrayEqualityCheck(crazyOrder,
+ new FieldsFromArray<Integer>(Integer.class, 4, 1, 2).extract(testIntArray));
+
+ }
+
+ private void arrayEqualityCheck(Object[] array1, Object[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i]);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/05c15198/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
new file mode 100644
index 0000000..0379fe0
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/extractor/FieldsFromTupleTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple;
+import org.apache.flink.api.java.tuple.Tuple1;
+import org.apache.flink.api.java.tuple.Tuple10;
+import org.apache.flink.api.java.tuple.Tuple11;
+import org.apache.flink.api.java.tuple.Tuple12;
+import org.apache.flink.api.java.tuple.Tuple13;
+import org.apache.flink.api.java.tuple.Tuple14;
+import org.apache.flink.api.java.tuple.Tuple15;
+import org.apache.flink.api.java.tuple.Tuple16;
+import org.apache.flink.api.java.tuple.Tuple17;
+import org.apache.flink.api.java.tuple.Tuple18;
+import org.apache.flink.api.java.tuple.Tuple19;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple20;
+import org.apache.flink.api.java.tuple.Tuple21;
+import org.apache.flink.api.java.tuple.Tuple22;
+import org.apache.flink.api.java.tuple.Tuple23;
+import org.apache.flink.api.java.tuple.Tuple24;
+import org.apache.flink.api.java.tuple.Tuple25;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.tuple.Tuple4;
+import org.apache.flink.api.java.tuple.Tuple5;
+import org.apache.flink.api.java.tuple.Tuple6;
+import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.tuple.Tuple8;
+import org.apache.flink.api.java.tuple.Tuple9;
+import org.apache.flink.streaming.api.windowing.extractor.FieldsFromTuple;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FieldsFromTupleTest {
+
+ private double[] testDouble;
+
+ @Before
+ public void init() {
+ testDouble = new double[Tuple.MAX_ARITY];
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ testDouble[i] = i;
+ }
+ }
+
+ @Test
+ public void testUserSpecifiedOrder() throws InstantiationException, IllegalAccessException {
+ Tuple currentTuple = (Tuple) CLASSES[Tuple.MAX_ARITY - 1].newInstance();
+ for (int i = 0; i < Tuple.MAX_ARITY; i++) {
+ currentTuple.setField(testDouble[i], i);
+ }
+
+ double[] expected = { testDouble[5], testDouble[3], testDouble[6], testDouble[7],
+ testDouble[0] };
+ arrayEqualityCheck(expected, new FieldsFromTuple(5, 3, 6, 7, 0).extract(currentTuple));
+
+ double[] expected2 = { testDouble[0], testDouble[Tuple.MAX_ARITY - 1] };
+ arrayEqualityCheck(expected2,
+ new FieldsFromTuple(0, Tuple.MAX_ARITY - 1).extract(currentTuple));
+
+ double[] expected3 = { testDouble[Tuple.MAX_ARITY - 1], testDouble[0] };
+ arrayEqualityCheck(expected3,
+ new FieldsFromTuple(Tuple.MAX_ARITY - 1, 0).extract(currentTuple));
+
+ double[] expected4 = { testDouble[13], testDouble[4], testDouble[5], testDouble[4],
+ testDouble[2], testDouble[8], testDouble[6], testDouble[2], testDouble[8],
+ testDouble[3], testDouble[5], testDouble[2], testDouble[16], testDouble[4],
+ testDouble[3], testDouble[2], testDouble[6], testDouble[4], testDouble[7],
+ testDouble[4], testDouble[2], testDouble[8], testDouble[7], testDouble[2] };
+ arrayEqualityCheck(expected4, new FieldsFromTuple(13, 4, 5, 4, 2, 8, 6, 2, 8, 3, 5, 2, 16,
+ 4, 3, 2, 6, 4, 7, 4, 2, 8, 7, 2).extract(currentTuple));
+ }
+
+ private void arrayEqualityCheck(double[] array1, double[] array2) {
+ assertEquals("The result arrays must have the same length", array1.length, array2.length);
+ for (int i = 0; i < array1.length; i++) {
+ assertEquals("Unequal fields at position " + i, array1[i], array2[i], 0d);
+ }
+ }
+
+ private static final Class<?>[] CLASSES = new Class<?>[] { Tuple1.class, Tuple2.class,
+ Tuple3.class, Tuple4.class, Tuple5.class, Tuple6.class, Tuple7.class, Tuple8.class,
+ Tuple9.class, Tuple10.class, Tuple11.class, Tuple12.class, Tuple13.class,
+ Tuple14.class, Tuple15.class, Tuple16.class, Tuple17.class, Tuple18.class,
+ Tuple19.class, Tuple20.class, Tuple21.class, Tuple22.class, Tuple23.class,
+ Tuple24.class, Tuple25.class };
+
+}
[17/34] incubator-flink git commit: [streaming] Windowing API update
+ package refactor
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
new file mode 100644
index 0000000..953759c
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
+
+ /**
+ * Auto-generated serial version UID
+ */
+ private static final long serialVersionUID = -8038984294071650730L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(WindowInvokable.class);
+
+ private LinkedList<TriggerPolicy<IN>> triggerPolicies;
+ private LinkedList<EvictionPolicy<IN>> evictionPolicies;
+ private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
+ private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
+ private LinkedList<Thread> activePolicyTreads = new LinkedList<Thread>();
+ protected LinkedList<IN> buffer = new LinkedList<IN>();
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+
+ /**
+ * This constructor created a windowing invokable using trigger and eviction
+ * policies.
+ *
+ * @param userFunction
+ * The user defined {@link ReduceFunction}
+ * @param triggerPolicies
+ * A list of {@link TriggerPolicy}s and/or
+ * {@link ActiveTriggerPolicy}s
+ * @param evictionPolicies
+ * A list of {@link EvictionPolicy}s and/or
+ * {@link ActiveEvictionPolicy}s
+ */
+ public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction);
+
+ this.triggerPolicies = triggerPolicies;
+ this.evictionPolicies = evictionPolicies;
+
+ activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+ for (TriggerPolicy<IN> tp : triggerPolicies) {
+ if (tp instanceof ActiveTriggerPolicy) {
+ activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
+ }
+ }
+
+ activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+ for (EvictionPolicy<IN> ep : evictionPolicies) {
+ if (ep instanceof ActiveEvictionPolicy) {
+ activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
+ }
+ }
+ }
+
+ @Override
+ public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
+ super.open(parameters);
+ for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
+ Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+ if (target != null) {
+ Thread thread = new Thread(target);
+ activePolicyTreads.add(thread);
+ thread.start();
+ }
+ }
+ };
+
+ /**
+ * This class allows the active trigger threads to call back and push fake
+ * elements at any time.
+ */
+ private class WindowingCallback implements ActiveTriggerCallback {
+ private ActiveTriggerPolicy<IN> policy;
+
+ public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+ this.policy = policy;
+ }
+
+ @Override
+ public void sendFakeElement(Object datapoint) {
+ processFakeElement(datapoint, this.policy);
+ }
+
+ }
+
+ @Override
+ protected void immutableInvoke() throws Exception {
+
+ // Prevent empty data streams
+ if ((reuse = recordIterator.next(reuse)) == null) {
+ throw new RuntimeException("DataStream must not be empty");
+ }
+
+ // Continuously run
+ while (reuse != null) {
+ processRealElement(reuse.getObject());
+
+ // Recreate the reuse-StremRecord object and load next StreamRecord
+ resetReuse();
+ reuse = recordIterator.next(reuse);
+ }
+
+ // Stop all remaining threads from policies
+ for (Thread t : activePolicyTreads) {
+ t.interrupt();
+ }
+
+ // finally trigger the buffer.
+ emitFinalWindow(null);
+
+ }
+
+ @Override
+ protected void mutableInvoke() throws Exception {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("There is currently no mutable implementation of this operator. Immutable version is used.");
+ }
+ immutableInvoke();
+ }
+
+ /**
+ * This method gets called in case of an grouped windowing in case central
+ * trigger occurred and the arriving element causing the trigger is not part
+ * of this group.
+ *
+ * Remark: This is NOT the same as
+ * {@link WindowInvokable#processFakeElement(Object, TriggerPolicy)}!
+ * Here the eviction using active policies takes place after the call to the
+ * UDF. Usually it is done before when fake elements get submitted. This
+ * special behaviour is needed to allow the
+ * {@link GroupedWindowInvokable} to send central triggers to all groups,
+ * even if the current element does not belong to the group.
+ *
+ * @param input
+ * a fake input element
+ * @param policies
+ * the list of policies which caused the call with this fake
+ * element
+ */
+ protected synchronized void externalTriggerFakeElement(IN input,
+ List<TriggerPolicy<IN>> policies) {
+
+ // Set the current triggers
+ currentTriggerPolicies.addAll(policies);
+
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+ }
+
+ /**
+ * This method processed an arrived fake element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowInvokable#processRealElement(Object)}
+ *
+ * @param input
+ * a fake input element
+ * @param currentPolicy
+ * the policy which produced this fake element
+ */
+ protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+
+ // Set the current trigger
+ currentTriggerPolicies.add(currentPolicy);
+
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+ }
+
+ /**
+ * This method processed an arrived real element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowInvokable#processFakeElement(Object)}.
+ *
+ * @param input
+ * a real input element
+ * @param triggerPolicies
+ * Allows to set trigger policies which are maintained
+ * externally. This is the case for central policies in
+ * {@link GroupedWindowInvokable}.
+ */
+ protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
+ this.currentTriggerPolicies.addAll(triggerPolicies);
+ processRealElement(input);
+ }
+
+ /**
+ * This method processed an arrived real element The method is synchronized
+ * to ensure that it cannot interleave with
+ * {@link WindowInvokable#processFakeElement(Object)}
+ *
+ * @param input
+ * a real input element
+ */
+ protected synchronized void processRealElement(IN input) {
+
+ // Run the precalls to detect missed windows
+ for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
+ // Remark: In case multiple active triggers are present the ordering
+ // of the different fake elements returned by this triggers becomes
+ // a problem. This might lead to unexpected results...
+ // Should we limit the number of active triggers to 0 or 1?
+ Object[] result = trigger.preNotifyTrigger(input);
+ for (Object in : result) {
+ processFakeElement(in, trigger);
+ }
+ }
+
+ // Remember if a trigger occurred
+ boolean isTriggered = false;
+
+ // Process the triggers
+ for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
+ if (triggerPolicy.notifyTrigger(input)) {
+ currentTriggerPolicies.add(triggerPolicy);
+ }
+ }
+
+ // call user function
+ if (!currentTriggerPolicies.isEmpty()) {
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+
+ // remember trigger
+ isTriggered = true;
+ }
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+
+ for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+
+ // Add the current element to the buffer
+ buffer.add(input);
+
+ }
+
+ /**
+ * This method does the final reduce at the end of the stream and emits the
+ * result.
+ *
+ * @param centralTriggerPolicies
+ * Allows to set trigger policies which are maintained
+ * externally. This is the case for central policies in
+ * {@link GroupedWindowInvokable}.
+ */
+ protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
+ if (!buffer.isEmpty()) {
+ currentTriggerPolicies.clear();
+
+ if (centralTriggerPolicies != null) {
+ currentTriggerPolicies.addAll(centralTriggerPolicies);
+ }
+
+ for (TriggerPolicy<IN> policy : triggerPolicies) {
+ currentTriggerPolicies.add(policy);
+ }
+
+ callUserFunctionAndLogException();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
new file mode 100644
index 0000000..b6456e1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public class WindowReduceInvokable<IN> extends WindowInvokable<IN, IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ ReduceFunction<IN> reducer;
+
+ public WindowReduceInvokable(ReduceFunction<IN> userFunction,
+ LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction, triggerPolicies, evictionPolicies);
+ this.reducer = userFunction;
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ Iterator<IN> reducedIterator = buffer.iterator();
+ IN reduced = null;
+
+ while (reducedIterator.hasNext() && reduced == null) {
+ reduced = reducedIterator.next();
+ }
+
+ while (reducedIterator.hasNext()) {
+ IN next = reducedIterator.next();
+ if (next != null) {
+ reduced = reducer.reduce(reduced, next);
+ }
+ }
+ if (reduced != null) {
+ collector.collect(reduced);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java
deleted file mode 100644
index 2752fd1..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.LinkedList;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public class WindowingGroupInvokable<IN, OUT> extends WindowingInvokable<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
- GroupReduceFunction<IN, OUT> reducer;
-
- public WindowingGroupInvokable(GroupReduceFunction<IN, OUT> userFunction,
- LinkedList<TriggerPolicy<IN>> triggerPolicies,
- LinkedList<EvictionPolicy<IN>> evictionPolicies) {
- super(userFunction, triggerPolicies, evictionPolicies);
- this.reducer = userFunction;
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- reducer.reduce(buffer, collector);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
deleted file mode 100644
index 95a999c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class WindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
-
- /**
- * Auto-generated serial version UID
- */
- private static final long serialVersionUID = -8038984294071650730L;
-
- private static final Logger LOG = LoggerFactory.getLogger(WindowingInvokable.class);
-
- private LinkedList<TriggerPolicy<IN>> triggerPolicies;
- private LinkedList<EvictionPolicy<IN>> evictionPolicies;
- private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
- private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
- private LinkedList<Thread> activePolicyTreads = new LinkedList<Thread>();
- protected LinkedList<IN> buffer = new LinkedList<IN>();
- private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-
- /**
- * This constructor created a windowing invokable using trigger and eviction
- * policies.
- *
- * @param userFunction
- * The user defined {@link ReduceFunction}
- * @param triggerPolicies
- * A list of {@link TriggerPolicy}s and/or
- * {@link ActiveTriggerPolicy}s
- * @param evictionPolicies
- * A list of {@link EvictionPolicy}s and/or
- * {@link ActiveEvictionPolicy}s
- */
- public WindowingInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
- LinkedList<EvictionPolicy<IN>> evictionPolicies) {
- super(userFunction);
-
- this.triggerPolicies = triggerPolicies;
- this.evictionPolicies = evictionPolicies;
-
- activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
- for (TriggerPolicy<IN> tp : triggerPolicies) {
- if (tp instanceof ActiveTriggerPolicy) {
- activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
- }
- }
-
- activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
- for (EvictionPolicy<IN> ep : evictionPolicies) {
- if (ep instanceof ActiveEvictionPolicy) {
- activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
- }
- }
- }
-
- @Override
- public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
- super.open(parameters);
- for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
- Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
- if (target != null) {
- Thread thread = new Thread(target);
- activePolicyTreads.add(thread);
- thread.start();
- }
- }
- };
-
- /**
- * This class allows the active trigger threads to call back and push fake
- * elements at any time.
- */
- private class WindowingCallback implements ActiveTriggerCallback {
- private ActiveTriggerPolicy<IN> policy;
-
- public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
- this.policy = policy;
- }
-
- @Override
- public void sendFakeElement(Object datapoint) {
- processFakeElement(datapoint, this.policy);
- }
-
- }
-
- @Override
- protected void immutableInvoke() throws Exception {
-
- // Prevent empty data streams
- if ((reuse = recordIterator.next(reuse)) == null) {
- throw new RuntimeException("DataStream must not be empty");
- }
-
- // Continuously run
- while (reuse != null) {
- processRealElement(reuse.getObject());
-
- // Recreate the reuse-StremRecord object and load next StreamRecord
- resetReuse();
- reuse = recordIterator.next(reuse);
- }
-
- // Stop all remaining threads from policies
- for (Thread t : activePolicyTreads) {
- t.interrupt();
- }
-
- // finally trigger the buffer.
- emitFinalWindow(null);
-
- }
-
- @Override
- protected void mutableInvoke() throws Exception {
- if (LOG.isInfoEnabled()) {
- LOG.info("There is currently no mutable implementation of this operator. Immutable version is used.");
- }
- immutableInvoke();
- }
-
- /**
- * This method gets called in case of an grouped windowing in case central
- * trigger occurred and the arriving element causing the trigger is not part
- * of this group.
- *
- * Remark: This is NOT the same as
- * {@link WindowingInvokable#processFakeElement(Object, TriggerPolicy)}!
- * Here the eviction using active policies takes place after the call to the
- * UDF. Usually it is done before when fake elements get submitted. This
- * special behaviour is needed to allow the
- * {@link GroupedWindowingInvokable} to send central triggers to all groups,
- * even if the current element does not belong to the group.
- *
- * @param input
- * a fake input element
- * @param policies
- * the list of policies which caused the call with this fake
- * element
- */
- protected synchronized void externalTriggerFakeElement(IN input,
- List<TriggerPolicy<IN>> policies) {
-
- // Set the current triggers
- currentTriggerPolicies.addAll(policies);
-
- // emit
- callUserFunctionAndLogException();
-
- // clear the flag collection
- currentTriggerPolicies.clear();
-
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
- for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
-
- for (int i = 0; i < currentMaxEviction; i++) {
- try {
- buffer.removeFirst();
- } catch (NoSuchElementException e) {
- // In case no more elements are in the buffer:
- // Prevent failure and stop deleting.
- break;
- }
- }
- }
-
- /**
- * This method processed an arrived fake element The method is synchronized
- * to ensure that it cannot interleave with
- * {@link WindowingInvokable#processRealElement(Object)}
- *
- * @param input
- * a fake input element
- * @param currentPolicy
- * the policy which produced this fake element
- */
- protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
-
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
- for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
-
- for (int i = 0; i < currentMaxEviction; i++) {
- try {
- buffer.removeFirst();
- } catch (NoSuchElementException e) {
- // In case no more elements are in the buffer:
- // Prevent failure and stop deleting.
- break;
- }
- }
-
- // Set the current trigger
- currentTriggerPolicies.add(currentPolicy);
-
- // emit
- callUserFunctionAndLogException();
-
- // clear the flag collection
- currentTriggerPolicies.clear();
- }
-
- /**
- * This method processed an arrived real element The method is synchronized
- * to ensure that it cannot interleave with
- * {@link WindowingInvokable#processFakeElement(Object)}.
- *
- * @param input
- * a real input element
- * @param triggerPolicies
- * Allows to set trigger policies which are maintained
- * externally. This is the case for central policies in
- * {@link GroupedWindowingInvokable}.
- */
- protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
- this.currentTriggerPolicies.addAll(triggerPolicies);
- processRealElement(input);
- }
-
- /**
- * This method processed an arrived real element The method is synchronized
- * to ensure that it cannot interleave with
- * {@link WindowingInvokable#processFakeElement(Object)}
- *
- * @param input
- * a real input element
- */
- protected synchronized void processRealElement(IN input) {
-
- // Run the precalls to detect missed windows
- for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
- // Remark: In case multiple active triggers are present the ordering
- // of the different fake elements returned by this triggers becomes
- // a problem. This might lead to unexpected results...
- // Should we limit the number of active triggers to 0 or 1?
- Object[] result = trigger.preNotifyTrigger(input);
- for (Object in : result) {
- processFakeElement(in, trigger);
- }
- }
-
- // Remember if a trigger occurred
- boolean isTriggered = false;
-
- // Process the triggers
- for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
- if (triggerPolicy.notifyTrigger(input)) {
- currentTriggerPolicies.add(triggerPolicy);
- }
- }
-
- // call user function
- if (!currentTriggerPolicies.isEmpty()) {
- // emit
- callUserFunctionAndLogException();
-
- // clear the flag collection
- currentTriggerPolicies.clear();
-
- // remember trigger
- isTriggered = true;
- }
-
- // Process the evictions and take care of double evictions
- // In case there are multiple eviction policies present,
- // only the one with the highest return value is recognized.
- int currentMaxEviction = 0;
-
- for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
- // use temporary variable to prevent multiple calls to
- // notifyEviction
- int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
- if (tmp > currentMaxEviction) {
- currentMaxEviction = tmp;
- }
- }
-
- for (int i = 0; i < currentMaxEviction; i++) {
- try {
- buffer.removeFirst();
- } catch (NoSuchElementException e) {
- // In case no more elements are in the buffer:
- // Prevent failure and stop deleting.
- break;
- }
- }
-
- // Add the current element to the buffer
- buffer.add(input);
-
- }
-
- /**
- * This method does the final reduce at the end of the stream and emits the
- * result.
- *
- * @param centralTriggerPolicies
- * Allows to set trigger policies which are maintained
- * externally. This is the case for central policies in
- * {@link GroupedWindowingInvokable}.
- */
- protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
- if (!buffer.isEmpty()) {
- currentTriggerPolicies.clear();
-
- if (centralTriggerPolicies != null) {
- currentTriggerPolicies.addAll(centralTriggerPolicies);
- }
-
- for (TriggerPolicy<IN> policy : triggerPolicies) {
- currentTriggerPolicies.add(policy);
- }
-
- callUserFunctionAndLogException();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java
deleted file mode 100644
index 89f98e0..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public class WindowingReduceInvokable<IN> extends WindowingInvokable<IN, IN> {
-
- private static final long serialVersionUID = 1L;
-
- ReduceFunction<IN> reducer;
-
- public WindowingReduceInvokable(ReduceFunction<IN> userFunction,
- LinkedList<TriggerPolicy<IN>> triggerPolicies,
- LinkedList<EvictionPolicy<IN>> evictionPolicies) {
- super(userFunction, triggerPolicies, evictionPolicies);
- this.reducer = userFunction;
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- Iterator<IN> reducedIterator = buffer.iterator();
- IN reduced = null;
-
- while (reducedIterator.hasNext() && reduced == null) {
- reduced = reducedIterator.next();
- }
-
- while (reducedIterator.hasNext()) {
- IN next = reducedIterator.next();
- if (next != null) {
- reduced = reducer.reduce(reduced, next);
- }
- }
- if (reduced != null) {
- collector.collect(reduced);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
index 0c2d691..132b495 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
@@ -17,10 +17,10 @@
package org.apache.flink.streaming.api.windowing.policy;
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowingInvokable;
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
/**
- * When used in {@link GroupedWindowingInvokable}, eviction policies must
+ * When used in {@link GroupedWindowInvokable}, eviction policies must
* provide a clone method. Eviction policies get cloned to provide an own
* instance for each group and respectively each individual element buffer as
* groups maintain their own buffers with the elements belonging to the
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
index 64f8bf3..f5772a1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
@@ -17,10 +17,10 @@
package org.apache.flink.streaming.api.windowing.policy;
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowingInvokable;
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
/**
- * When used in {@link GroupedWindowingInvokable}, trigger policies can provide
+ * When used in {@link GroupedWindowInvokable}, trigger policies can provide
* a clone method. Cloneable triggers can can be used in a distributed manner,
* which means they get cloned to provide an own instance for each group. This
* allows each group to trigger individually and only based on the elements
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
index 79288d1..9d5744b 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
@@ -77,7 +77,7 @@ public class GroupedWindowingInvokableTest {
LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
- GroupedWindowingInvokable<Integer, Integer> invokable = new GroupedWindowingInvokable<Integer, Integer>(
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
new ReduceFunction<Integer>() {
private static final long serialVersionUID = 1L;
@@ -136,7 +136,7 @@ public class GroupedWindowingInvokableTest {
LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
- GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
new ReduceFunction<Tuple2<Integer, String>>() {
private static final long serialVersionUID = 1L;
@@ -256,7 +256,7 @@ public class GroupedWindowingInvokableTest {
LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
- GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+ GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
distributedTriggers, evictions, triggers);
@@ -319,7 +319,7 @@ public class GroupedWindowingInvokableTest {
}
};
- GroupedWindowingInvokable<Integer, Integer> invokable = new GroupedWindowingInvokable<Integer, Integer>(
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
myReduceFunction, new KeySelector<Integer, Integer>() {
private static final long serialVersionUID = 1L;
@@ -395,7 +395,7 @@ public class GroupedWindowingInvokableTest {
}
};
- GroupedWindowingInvokable<Integer, Integer> invokable = new GroupedWindowingInvokable<Integer, Integer>(
+ GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
myReduceFunction, new KeySelector<Integer, Integer>() {
private static final long serialVersionUID = 1L;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/2d73c3f1/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
index 32e71ba..4c6b187 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
@@ -94,7 +94,7 @@ public class WindowingInvokableTest {
// Always delete all elements older then 4
evictions.add(new TimeEvictionPolicy<Integer>(4L, myTimeStamp));
- WindowingInvokable<Integer, Integer> invokable = new WindowingReduceInvokable<Integer>(
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
myReduceFunction, triggers, evictions);
ArrayList<Integer> result = new ArrayList<Integer>();
@@ -138,7 +138,7 @@ public class WindowingInvokableTest {
// time after the 3rd element
evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
- WindowingInvokable<Integer, Integer> invokable = new WindowingReduceInvokable<Integer>(
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
myReduceFunction, triggers, evictions);
List<Integer> expected = new ArrayList<Integer>();
@@ -191,7 +191,7 @@ public class WindowingInvokableTest {
// time after on the 5th element
evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
- WindowingInvokable<Integer, Integer> invokable2 = new WindowingReduceInvokable<Integer>(
+ WindowInvokable<Integer, Integer> invokable2 = new WindowReduceInvokable<Integer>(
myReduceFunction, triggers, evictions);
List<Integer> expected2 = new ArrayList<Integer>();
@@ -249,7 +249,7 @@ public class WindowingInvokableTest {
}
};
- WindowingInvokable<Integer, Integer> invokable = new WindowingReduceInvokable<Integer>(
+ WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
myReduceFunction, triggers, evictions);
ArrayList<Integer> result = new ArrayList<Integer>();
[33/34] incubator-flink git commit: [streaming] Updated javadocs for
new windowing semantics
Posted by mb...@apache.org.
[streaming] Updated javadocs for new windowing semantics
[streaming] Updated streaming-guide to match recent api changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/4e046a9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/4e046a9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/4e046a9b
Branch: refs/heads/master
Commit: 4e046a9bd43428820a409a0777ab281e3c366af5
Parents: 3f74e05
Author: Gyula Fora <gy...@apache.org>
Authored: Tue Dec 2 23:39:47 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:47:18 2014 +0100
----------------------------------------------------------------------
docs/streaming_guide.md | 189 ++++++++----
.../streaming/api/datastream/DataStream.java | 63 ++--
.../datastream/SingleOutputStreamOperator.java | 40 ++-
.../api/datastream/WindowedDataStream.java | 289 ++++++++++---------
.../streaming/api/windowing/helper/Count.java | 6 +-
.../streaming/api/windowing/helper/Delta.java | 9 +-
.../streaming/api/windowing/helper/Time.java | 10 +-
.../windowing/TimeWindowingExample.java | 2 +-
8 files changed, 360 insertions(+), 248 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index a0bc1d6..6038cd3 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -11,12 +11,12 @@ Introduction
------------
-Flink Streaming is an extension of the core Flink API for high-throughput, low-latency data stream processing. The system can connect to and process data streams from many data sources like Flume, Twitter, ZeroMQ and also from any user defined data source. Data streams can be transformed and modified using high-level functions similar to the ones provided by the batch processing API. Flink Streaming provides native support for iterative stream processing. The processed data can be pushed to different output types.
+Flink Streaming is an extension of the core Flink API for high-throughput, low-latency data stream processing. The system can connect to and process data streams from many data sources like RabbitMQ, Flume, Twitter, ZeroMQ and also from any user defined data source. Data streams can be transformed and modified using high-level functions similar to the ones provided by the batch processing API. Flink Streaming provides native support for iterative stream processing. The processed data can be pushed to different output types.
Flink Streaming API
-----------
-The Streaming API is part of the *addons* Maven project. All relevant classes are located in the *org.apache.flink.streaming* package.
+The Streaming API is currently part of the *addons* Maven project. All relevant classes are located in the *org.apache.flink.streaming* package.
Add the following dependency to your `pom.xml` to use the Flink Streaming.
@@ -91,14 +91,16 @@ As these steps are basically the same as in the core API we will only note the i
For stream processing jobs, the user needs to obtain a `StreamExecutionEnvironment` in contrast with the batch API where one would need an `ExecutionEnvironment`. The process otherwise is essentially the same:
~~~java
-StreamExecutionEnvironment.createLocalEnvironment(params…)
-StreamExecutionEnvironment.createRemoteEnvironment(params…)
+StreamExecutionEnvironment.getExecutionEnvironment()
+StreamExecutionEnvironment.createLocalEnvironment(parallelism)
+StreamExecutionEnvironment.createRemoteEnvironment(…)
~~~
For connecting to data streams the `StreamExecutionEnvironment` has many different methods, from basic file sources to completely general user defined data sources. We will go into details in the [basics](#basics) section.
~~~java
env.socketTextStream(host, port)
+env.fromElements(elements…)
~~~
After defining the data stream sources, the user can specify transformations on the data streams to create a new data stream. Different data streams can be also combined together for joint transformations which are being showcased in the [operations](#operations) section.
@@ -111,6 +113,7 @@ The processed data can be pushed to different outputs called sinks. The user can
~~~java
dataStream.writeAsCsv(path)
+dataStream.print()
~~~
Once the complete program is specified `execute(programName)` is to be called on the `StreamExecutionEnvironment`. This will either execute on the local machine or submit the program for execution on a cluster, depending on the chosen execution environment.
@@ -126,22 +129,22 @@ Basics
### DataStream
-The `DataStream` is the basic abstraction provided by the Flink Streaming API. It represents a continuous stream of data of a certain type from either a data source or a transformed data stream. Operations will be applied on individual data points or windows of the `DataStream` based on the type of the operation. For example the map operator transforms each data point individually while window or batch aggregations work on an interval of data points at the same time.
+The `DataStream` is the basic abstraction provided by the Flink Streaming API. It represents a continuous stream of data of a certain type from either a data source or a transformed data stream. Operations will be applied on individual data points or windows of the `DataStream` based on the type of the operation. For example the map operator transforms each data point individually while window operations work on an interval of data points at the same time.
-The operations may return different `DataStream` types allowing more elaborate transformations, for example the `groupBy()` method returns a `GroupedDataStream` which can be used for group operations.
+The operations may return different `DataStream` types allowing more elaborate transformations, for example the `groupBy(…)` method returns a `GroupedDataStream` which can be used for grouped operations such as aggregating by key.
### Partitioning
Partitioning controls how individual data points are distributed among the parallel instances of the transformation operators. By default *Forward* partitioning is used. There are several partitioning types supported in Flink Streaming:
- * *Forward*: Forward partitioning directs the output data to the next operator on the same machine (if possible) avoiding expensive network I/O. This is the default partitioner.
+ * *Forward*: Forward partitioning directs the output data to the next operator on the same machine (if possible) avoiding expensive network I/O. If there are more processing nodes than inputs or vice verse the load is distributed among the extra nodes in a round-robin fashion. This is the default partitioner.
Usage: `dataStream.forward()`
- * *Shuffle*: Shuffle partitioning randomly partitions the output data stream to the next operator using uniform distribution.
+ * *Shuffle*: Shuffle partitioning randomly partitions the output data stream to the next operator using uniform distribution. Use this only when it is important that the partitioning is randomised. If you only care about an even load use *Distribute*
Usage: `dataStream.shuffle()`
* *Distribute*: Distribute partitioning directs the output data stream to the next operator in a round-robin fashion, achieving a balanced distribution.
Usage: `dataStream.distribute()`
- * *Field*: Field partitioning partitions the output data stream based on the hash code of a selected key field. Data points with the same key are directed to the same operator instance.
-Usage: `dataStream.partitionBy(keyposition)`
+ * *Field/Key*: Field/Key partitioning partitions the output data stream based on the hash code of a selected key of the tuples. Data points with the same key are directed to the same operator instance. The user can define keys by field positions (for tuple and array types), field expressions (for Pojo types) and custom keys using the `KeySelector` interface.
+Usage: `dataStream.partitionBy(keys)`
* *Broadcast*: Broadcast partitioning sends the output data stream to all parallel instances of the next operator.
Usage: `dataStream.broadcast()`
* *Global*: All data points end up at the same operator instance. To achieve this use the parallelism setting of the corresponding operator.
@@ -149,22 +152,23 @@ Usage: `operator.setParallelism(1)`
### Sources
-The user can connect to data streams by the different implementations of `DataStreamSource` using methods provided by the `StreamExecutionEnvironment`. There are several predefined ones similar to the ones of the batch API and some streaming specific ones like:
+The user can connect to data streams by the different implementations of `SourceFunction` using `StreamExecutionEnvironment.addSource(SourceFunction)`. In contrast with other operators, DataStreamSources have a default operator parallelism of 1 which can be increased using the `.setParallelism(dop)` method as we will see later at the operator settings.
+
+There are several predefined ones similar to the ones of the batch API and some streaming specific ones like:
* `socketTextStream(hostname, port)`
* `readTextStream(filepath)`
- * `genereateSequence(from, to)`
+ * `generateSequence(from, to)`
* `fromElements(elements…)`
* `fromCollection(collection)`
* `readTextFile(filepath)`
These can be used to easily test and debug streaming programs.
There are pre-implemented connectors for a number of the most popular message queue services, please refer to the section on [connectors](#stream-connectors) for more detail.
-Besides the pre-defined solutions the user can implement their own source by implementing the `SourceFunction` interface and using the `addSource(sourceFunction)` method of the `StreamExecutionEnvironment`.
### Sinks
-`DataStreamSink` represents the different outputs of a Flink Streaming program. There are several pre-defined implementations `DataStreamSink` available right away:
+`DataStreamSink` represents the different outputs of a Flink Streaming program. There are several pre-defined implementations available right away:
* `dataStream.print()` – Writes the DataStream to the standard output, practical for testing purposes
* `dataStream.writeAsText(parameters)` – Writes the DataStream to a text file
@@ -238,7 +242,7 @@ dataStream.reduce(new ReduceFunction<Integer>() {
~~~
#### Merge
-Merges two or more `DataStream` instances creating a new DataStream containing all the elements from all the streams.
+Merges two or more `DataStream` outputs, creating a new DataStream containing all the elements from all the streams.
~~~java
dataStream.merge(otherStream1, otherStream2…)
@@ -246,67 +250,141 @@ dataStream.merge(otherStream1, otherStream2…)
### Grouped operators
-Some transformations require that the `DataStream` is grouped on some key value. The user can create a `GroupedDataStream` by calling the `groupBy(keyPosition)` method of a non-grouped `DataStream`. The user can apply different reduce transformations on the obtained `GroupedDataStream`:
+Some transformations require that the elements of a `DataStream` are grouped on some key. The user can create a `GroupedDataStream` by calling the `groupBy(key)` method of a non-grouped `DataStream`.
+Keys can be of three types: fields positions (applicable for tuple/array types), field expressions (applicable for pojo types), KeySelector instances.
+
+The user can apply different reduce transformations on the obtained `GroupedDataStream`:
#### Reduce on GroupedDataStream
When the reduce operator is applied on a grouped data stream, the user-defined `ReduceFunction` will combine subsequent pairs of elements having the same key value. The combined results are sent to the output stream.
### Aggregations
-The Flink Streaming API supports different types of aggregation operators similarly to the core API. For grouped data streams the aggregations work in a grouped fashion.
+The Flink Streaming API supports different types of pre-defined aggregation operators similarly to the core API.
+
+Types of aggregations: `sum(field)`, `min(field)`, `max(field)`, `minBy(field, first)`, `maxBy(field, first)`
+
+With `sum`, `min`, and `max` for every incoming tuple the selected field is replaced with the current aggregated value. Fields can be selected using either field positions or field expressions (similarly to grouping).
+
+With `minBy` and `maxBy` the output of the operator is the element with the current minimal or maximal value at the given field. If more components share the minimum or maximum value, the user can decide if the operator should return the first or last element. This can be set by the `first` boolean parameter.
+
+### Window operators
+
+Flink streaming provides very flexible windowing semantics to create arbitrary windows (also referred to as discretizations or slices) of the DataStreams and apply reduce or aggregation operations on them afterwards. Windowing can be used for instance to create rolling aggregations of the most recent N elements, where N could be defined by Time, Count or any arbitrary measure.
+
+The user can control the size (eviction) of the windows and the frequency of reduce/aggregation calls (triggers) on them in an intuitive api:
-Types of aggregations: `sum(fieldPosition)`, `min(fieldPosition)`, `max(fieldPosition)`, `minBy(fieldPosition, first)`, `maxBy(fieldPosition, first)`
+~~~java
+dataStream.window(…).every(…).reduce/reduceGroup/aggregation
+~~~
+
+The next example would create windows that hold elements of the last 5 seconds, and the user defined aggregation/reduce is executed on the windows every second (sliding the window by 1 second):
+
+~~~java
+dataStream.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(1, TimeUnit.SECONDS))
+~~~
-With `sum`, `min`, and `max` for every incoming tuple the selected field is replaced with the current aggregated value.
+This approach is often referred to as policy based windowing. Different policies (count, time, etc.) can be mixed as well; for example to downsample our stream, we can create a window that will take the latest 100 elements of our stream every minute:
-With `minBy` and `maxBy` the output of the operator is the element with the current minimal or maximal value at the given fieldposition. If more components share the minimum or maximum value, the user can decide if the operator should return the first or last element. This can be set by the `first` boolean parameter.
+~~~java
+dataStream.window(Count.of(100)).every(Time.of(1, TimeUnit.MINUTES))
+~~~
-### Window/Batch operators
+The user can also omit the `.every(…)` call which results in a tumbling window, which empties the window after every aggregation call.
-Window and batch operators allow the user to execute function on slices or windows of the DataStream in a sliding fashion. If the stepsize for the slide is not defined then the window/batchsize is used as stepsize by default. The user can also use user defined timestamps for calculating time windows.
+Several predefined policies are provided in the API, including delta-based, count-based and time-based policies. These can be accessed through the static methods provided by the PolicyHelper classses:
+
+`Time.of(…)`
+`Count.of(…)`
+`Delta.of(…)`
+
+For detailed description of these policies please refer to the javadocs.
+
+Now we are going to take a deeper look at the policies.
+
+#### Policy based windowing
+The policy based windowing is a highly flexible way to specify your stream discretisation also called windowing semantics. Two types of policies are used for such a specification:
+
+1) `TriggerPolicy` This policy defines when to trigger the reduce UDF on the current window and emit the result. In the API it completes a window statement such as: `.window(..).every(...)`, while we pass the triggering policy within `every`.
+
+When multiple triggers are used, the reduce/aggregation will be triggered at every trigger.
+
+Several predefined policies are provided in the API, including delta-based, punctuation based, count-based and time-based policies. Policies are in general UDFs and can implement any custom behaviour as well.
+
+2) `Eviction Policy` This policy defines the length of a window as a means of a predicate for evicting tuples when they are no longer needed. In the API this can be defined by the `.window(..)` operation on a stream. There are mostly the same predefined policy types provided as for trigger policies.
+
+When multiple evictions are used the strictest one will control the elements in the window. For instance in the call `ds.window(Count.of(5), Time.of(1,TimeUnit.SECONDS)).every(..)` we will get a window of max 5 elements which have arrived in the last second.
+
+
+In addition to the `ds.window().every()` style users can specifically pass the list of trigger and eviction policies during the window call:
+~~~java
+myStream.window(ListOfTriggerPolicies,ListOfEvictionPolicies)
+~~~
-When applied to grouped data streams the data stream is batched/windowed for different key values separately.
+By default most triggers can only trigger when a new element arrives. This might not be suitable for all the use-cases, especially when time based windowing is used. You may want to trigger not only when an element arrives but also in between. Active policies provide this functionality. The predefined time-based policies are already implemented in such an active way and can hold as an example in case you want to implement your own user defined active policy.
-For example a `dataStream.groupBy(0).batch(100, 10)` produces batches of the last 100 elements for each key value with 10 record step size.
+Time-based trigger and eviction policies can work with user defined `TimeStamp` implementations, these policies already cover most use cases.
-#### Reduce on windowed/batched data streams
-The transformation calls a user-defined `ReduceFunction` on records received in the batch or during the predefined time window. The window is shifted after each reduce call. The user can also use the different streaming aggregations.
+#### Reduce on windowed data streams
+The transformation calls a user-defined `ReduceFunction` at every trigger on the records currently in the window. The user can also use the different streaming aggregations.
A window reduce that sums the elements in the last minute with 10 seconds slide interval:
~~~java
-dataStream.window(60000, 10000).sum();
+dataStream.window(Time.of(1, TimeUnit.MINUTES)).every(Time.of(10,TimeUnit.SECONDS)).sum(field);
~~~
-#### ReduceGroup on windowed/batched data streams
-The transformation calls a `GroupReduceFunction` for each data batch or data window. The batch/window slides by the predefined number of elements/time after each call.
+#### ReduceGroup on windowed data streams
+The transformation calls a `GroupReduceFunction` for each data batch or data window similarly as a reduce, but providing access to all elements in the window.
~~~java
-dataStream.batch(1000, 100).reduceGroup(reducer);
+dataStream.window(…).every(…).reduceGroup(reducer);
~~~
-#### Policy based windowing
-The policy based windowing is an highly flexible way to specify your stream discretisation also called windowing semantics. Two types of policies are used for such a specification:
+#### Grouped operations on windowed data streams
+Calling the `.groupBy(fields)` method on a windowed stream groups the elements by the given fields inside the windows. The window sizes (evictions) and slide sizes (triggers) will be calculated on the whole stream (in a central fashion), but the user defined functions will be applied on a per group basis.
-1) `TriggerPolicy` This policy defines when to trigger the reduce UDF on the current window and emit the result. In the API it completes a window statement such as: `.window(..).every(...)`, while we pass the triggering policy within `every`. Several predefind policies are provided in the API, including delta-based, punctuation based, count-based and time-based policies. Anyhow, policies are in general UDFs and can implement any custom behaviour as well.
+The user can also create windows and triggers on a per group basis calling `.window(…).every(…)` on an already grouped data stream. To highlight the differences let us look at to examples.
-2) `Eviction Policy` This policy defines the length of a window as a means of a predicate for evicting tuples when they are no longer needed. In the API this can be defined by the `.window(..)` operation on a stream. There are mostely the same predefined policy types provided as for trigger policies.
+To get the maximal value by key on the last 100 elements we use the first approach:
+~~~java
+dataStream.window(Count.of(100)).every(…).groupBy(groupingField).max(field);
+~~~
+Using this approach we took the last 100 elements, divided it into groups by key then applied the aggregation.
-For example, if we want a fife tuples long sliding window that triggers every second we can write the following:
+To create fixed size windows for every key we need to reverse the order of the groupBy call. So to take the max for the last 100 elements in Each group:
~~~java
-myStream.window(Count.of(5)).every(Time.of(1,TimeUnit.SECOND))
+dataStream.groupBy(groupingField).window(Count.of(100)).every(…).max(field);
~~~
+This will create separate windows for different keys and apply the trigger and eviction policies on a per group basis.
+
+### Temporal database style operators
+
+While database style operators like joins (on key) and crosses are hard to define properly on data streams, a straight forward implementation is to apply these operators on windows of the data streams.
+
+Currently join and cross operators are supported on time windows.
+
+The Join transformation produces a new Tuple DataStream with two fields. Each tuple holds a joined element of the first input DataStream in the first tuple field and a matching element of the second input DataStream in the second field for the current window.
+
+The following code shows a default Join transformation using field position keys:
-In addition to this, multiple policies can be added to a `LinkedList` and then be used in parallel by writing the following:
~~~java
-myStream.window(ListOfTriggerPolicies,ListOfEvictionPolicies)
+dataStream1.join(dataStream2)
+ .onWindow(windowing_params)
+ .where(key_in_first)
+ .equalTo(key_in_second);
+~~~
+
+The Cross transformation combines two DataStreams into one DataStreams. It builds all pairwise combinations of the elements of both input DataStreams in the current window, i.e., it builds a temporal Cartesian product.
+
+~~~java
+dataStream1.cross(dataStream2).onWindow(windowing_params);
~~~
-Especially when time based windowing is used, you may want to trigger not only when an element arrives but also in between. Active policies provide this functionality. The predefined time-based policies are already implemented in such an active way and can hold as an example in case you want to implement your own user defined active policy. Anyhow, as our time-based trigger and eviction policies can work with user defined `TimeStamp` implementations, this policies already cover most use cases.
### Co operators
-Co operators allow the users to jointly transform two `DataStreams` of different types providing a simple way to jointly manipulate a shared state. It is designed to support joint stream transformations where merging is not appropriate due to different data types or the in cases when user needs explicit track of the datas origin.
+Co operators allow the users to jointly transform two `DataStreams` of different types providing a simple way to jointly manipulate a shared state. It is designed to support joint stream transformations where merging is not appropriate due to different data types or in case the user needs explicit tracking of the joined stream origin.
Co operators can be applied to `ConnectedDataStreams` which represent two `DataStreams` of possibly different types. A `ConnectedDataStream` can be created by calling the `connect(otherDataStream)` method of a `DataStream`. Please note that the two connected `DataStreams` can also be merged data streams.
#### Map on ConnectedDataStream
@@ -354,26 +432,8 @@ dataStream1.connect(dataStream2)
})
~~~
-#### windowReduceGroup on ConnectedDataStream
-The windowReduceGroup operator applies a user defined `CoGroupFunction` to time aligned windows of the two data streams and return zero or more elements of an arbitrary type. The user can define the window and slide intervals and can also implement custom timestamps to be used for calculating windows.
-
-~~~java
-DataStream<Integer> dataStream1 = ...
-DataStream<String> dataStream2 = ...
-
-dataStream1.connect(dataStream2)
- .windowReduceGroup(new CoGroupFunction<Integer, String, String>() {
-
- @Override
- public void coGroup(Iterable<Integer> first, Iterable<String> second,
- Collector<String> out) throws Exception {
-
- //Do something here
-
- }
- }, 10000, 5000);
-~~~
-
+#### windowReduce on ConnectedDataStream
+The windowReduce operator applies a user defined `CoWindowFunction` to time aligned windows of the two data streams and return zero or more elements of an arbitrary type. The user can define the window and slide intervals and can also implement custom timestamps to be used for calculating windows.
#### Reduce on ConnectedDataStream
The Reduce operator for the `ConnectedDataStream` applies a simple reduce transformation on the joined data streams and then maps the reduced elements to a common output type.
@@ -465,7 +525,9 @@ Setting parallelism for operators works exactly the same way as in the core Flin
### Buffer timeout
By default data points are not transferred on the network one-by-one, which would cause unnecessary network traffic, but are buffered in the output buffers. The size of the output buffers can be set in the Flink config files. While this method is good for optimizing throughput, it can cause latency issues when the incoming stream is not fast enough.
-To tackle this issue the user can call `env.setBufferTimeout(timeoutMillis)` on the execution environment (or on individual operators) to set a maximum wait time for the buffers to fill up. After this time the buffers are flushed automatically even if they are not full. Usage:
+To tackle this issue the user can call `env.setBufferTimeout(timeoutMillis)` on the execution environment (or on individual operators) to set a maximum wait time for the buffers to fill up. After this time the buffers are flushed automatically even if they are not full. The default value for this timeout is 100ms which should be appropriate for most use-cases.
+
+Usage:
~~~java
LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
@@ -474,8 +536,13 @@ env.setBufferTimeout(timeoutMillis);
env.genereateSequence(1,10).map(new MyMapper()).setBufferTimeout(timeoutMillis);
~~~
+To maximise the throughput the user can call `.setBufferTimeout(-1)` which will remove the timeout and buffers will only be flushed when they are full.
+To minimise latency, set the timeout to a value close to 0 (fro example 5 or 10 ms). Theoretically a buffer timeout of 0 will cause all outputs to be flushed when produced, but this setting should be avoided because it can cause severe performance degradation.
+
### Mutability
+This is currently a beta feature and it is only supported for a subset of the available operators.
+
Most operators allow setting mutability for reading input data. If the operator is set mutable then the variable used to store input data for operators will be reused in a mutable fashion to avoid excessive object creation. By default, all operators are set to immutable.
Usage:
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 55f5e71..f0e4309 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -595,6 +595,20 @@ public class DataStream<OUT> {
}
/**
+ * Applies an aggregation that gives the maximum of the data stream at the
+ * given position.
+ *
+ * @param positionToMax
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
+ checkFieldRange(positionToMax);
+ return aggregate(ComparableAggregator.getAggregator(positionToMax, getOutputType(),
+ AggregationType.MAX));
+ }
+
+ /**
* Applies an aggregation that that gives the maximum of the pojo data
* stream at the given field expression. A field expression is either the
* name of a public field or a getter method with parentheses of the
@@ -685,20 +699,6 @@ public class DataStream<OUT> {
}
/**
- * Applies an aggregation that gives the maximum of the data stream at the
- * given position.
- *
- * @param positionToMax
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
- checkFieldRange(positionToMax);
- return aggregate(ComparableAggregator.getAggregator(positionToMax, getOutputType(),
- AggregationType.MAX));
- }
-
- /**
* Applies an aggregation that that gives the current element with the
* maximum value at the given position, if more elements have the maximum
* value at the given position, the operator returns the first one by
@@ -745,16 +745,27 @@ public class DataStream<OUT> {
}
/**
- * This allows you to set up windowing through a nice API using
- * {@link WindowingHelper} such as {@link Time}, {@link Count} and
- * {@link Delta}. Windowing allows the user to apply different user defined
- * functions on predefined chunks of the data stream. For example a reducer
- * could be applied on every 5 seconds of data to count the elements in that
- * time window.
+ * Create a {@link WindowedDataStream} that can be used to apply
+ * transformation like {@link WindowedDataStream#reduce} or aggregations on
+ * preset chunks(windows) of the data stream. To define the windows one or
+ * more {@link WindowingHelper} such as {@link Time}, {@link Count} and
+ * {@link Delta} can be used.</br></br> When applied to a grouped data
+ * stream, the windows (evictions) and slide sizes (triggers) will be
+ * computed on a per group basis. </br></br> For more advanced control over
+ * the trigger and eviction policies please refer to
+ * {@link #window(triggers, evicters)} </br> </br> For example to create a
+ * sum every 5 seconds in a tumbling fashion:</br>
+ * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).sum(field)} </br></br> To
+ * create sliding windows use the
+ * {@link WindowedDataStream#every(WindowingHelper...)} </br></br> The same
+ * example with 3 second slides:</br>
+ *
+ * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(3,
+ * TimeUnit.SECONDS)).sum(field)}
*
* @param policyHelpers
* Any {@link WindowingHelper} such as {@link Time},
- * {@link Count} and {@link Delta}.
+ * {@link Count} and {@link Delta} to define the window.
* @return A {@link WindowedDataStream} providing further operations.
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
@@ -763,11 +774,11 @@ public class DataStream<OUT> {
}
/**
- * Set up a windowed data stream using the given {@link TriggerPolicy}s and
- * {@link EvictionPolicy}s. Windowing allows the user to apply different
- * user defined functions on predefined chunks of the data stream. For
- * example a reducer could be applied on every 5 seconds of data to count
- * the elements in that time window.
+ * Create a {@link WindowedDataStream} using the given {@link TriggerPolicy}
+ * s and {@link EvictionPolicy}s. Windowing can be used to apply
+ * transformation like {@link WindowedDataStream#reduce} or aggregations on
+ * preset chunks(windows) of the data stream.</br></br>For most common
+ * use-cases please refer to {@link #window(WindowingHelper...)}
*
* @param triggers
* The list of {@link TriggerPolicy}s that will determine how
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
index 294a56f..714807c 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/SingleOutputStreamOperator.java
@@ -23,6 +23,7 @@ import java.util.Map.Entry;
import org.apache.commons.lang3.SerializationException;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.collector.OutputSelector;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.streamvertex.StreamingRuntimeContext;
@@ -71,6 +72,7 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
}
/**
+ * This is a beta feature, use with care </br><br/>
* Sets the mutability of the operator. If the operator is set to mutable,
* the tuples received in the user defined functions, will be reused after
* the function call. Setting an operator to mutable reduces garbage
@@ -123,11 +125,11 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
}
/**
- * Register an operator state for this operator by the given name. This name
- * can be used to retrieve the state during runtime using
- * {@link StreamingRuntimeContext#getState(String)}. To obtain the
- * {@link StreamingRuntimeContext} from the user-defined function use the
- * {@link RichFunction#getRuntimeContext()} method.
+ * This is a beta feature </br></br> Register an operator state for this
+ * operator by the given name. This name can be used to retrieve the state
+ * during runtime using {@link StreamingRuntimeContext#getState(String)}. To
+ * obtain the {@link StreamingRuntimeContext} from the user-defined function
+ * use the {@link RichFunction#getRuntimeContext()} method.
*
* @param name
* The name of the operator state.
@@ -135,23 +137,23 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
* The state to be registered for this name.
* @return The data stream with state registered.
*/
- public SingleOutputStreamOperator<OUT, O> registerState(String name, OperatorState<?> state) {
+ protected SingleOutputStreamOperator<OUT, O> registerState(String name, OperatorState<?> state) {
jobGraphBuilder.addOperatorState(getId(), name, state);
return this;
}
/**
- * Register operator states for this operator provided in a map. The
- * registered states can be retrieved during runtime using
- * {@link StreamingRuntimeContext#getState(String)}. To obtain the
- * {@link StreamingRuntimeContext} from the user-defined function use the
- * {@link RichFunction#getRuntimeContext()} method.
+ * This is a beta feature </br></br> Register operator states for this
+ * operator provided in a map. The registered states can be retrieved during
+ * runtime using {@link StreamingRuntimeContext#getState(String)}. To obtain
+ * the {@link StreamingRuntimeContext} from the user-defined function use
+ * the {@link RichFunction#getRuntimeContext()} method.
*
* @param states
* The map containing the states that will be registered.
* @return The data stream with states registered.
*/
- public SingleOutputStreamOperator<OUT, O> registerState(Map<String, OperatorState<?>> states) {
+ protected SingleOutputStreamOperator<OUT, O> registerState(Map<String, OperatorState<?>> states) {
for (Entry<String, OperatorState<?>> entry : states.entrySet()) {
jobGraphBuilder.addOperatorState(getId(), entry.getKey(), entry.getValue());
}
@@ -160,8 +162,18 @@ public class SingleOutputStreamOperator<OUT, O extends SingleOutputStreamOperato
}
@SuppressWarnings("unchecked")
- public SingleOutputStreamOperator<OUT, O> partitionBy(int keyposition) {
- return (SingleOutputStreamOperator<OUT, O>) super.partitionBy(keyposition);
+ public SingleOutputStreamOperator<OUT, O> partitionBy(int... keypositions) {
+ return (SingleOutputStreamOperator<OUT, O>) super.partitionBy(keypositions);
+ }
+
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> partitionBy(String... fields) {
+ return (SingleOutputStreamOperator<OUT, O>) super.partitionBy(fields);
+ }
+
+ @SuppressWarnings("unchecked")
+ public SingleOutputStreamOperator<OUT, O> partitionBy(KeySelector<OUT, ?> keySelector) {
+ return (SingleOutputStreamOperator<OUT, O>) super.partitionBy(keySelector);
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 15e5110..43b3993 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.RichReduceFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
@@ -43,10 +44,10 @@ import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
import org.apache.flink.streaming.util.serialization.FunctionTypeWrapper;
/**
- * A {@link WindowedDataStream} represents a data stream whose elements are
- * batched together in a sliding batch. operations like
- * {@link #reduce(ReduceFunction)} or {@link #reduceGroup(GroupReduceFunction)}
- * are applied for each batch and the batch is slid afterwards.
+ * A {@link WindowedDataStream} represents a data stream that has been divided
+ * into windows (predefined chunks). User defined function such as
+ * {@link #reduce(ReduceFunction)}, {@link #reduceGroup(GroupReduceFunction)} or
+ * aggregations can be applied to the windows.
*
* @param <OUT>
* The output type of the {@link WindowedDataStream}
@@ -122,6 +123,20 @@ public class WindowedDataStream<OUT> {
this.allCentral = windowedDataStream.allCentral;
}
+ /**
+ * Defines the slide size (trigger frequency) for the windowed data stream.
+ * This controls how often the user defined function will be triggered on
+ * the window. </br></br> For example to get a window of 5 elements with a
+ * slide of 2 seconds use: </br></br>
+ * {@code ds.window(Count.of(5)).every(Time.of(2,TimeUnit.SECONDS))}
+ * </br></br> The user function in this case will be called on the 5 most
+ * recent elements every 2 seconds
+ *
+ * @param policyHelpers
+ * The policies that define the triggering frequency
+ *
+ * @return The windowed data stream with triggering set
+ */
@SuppressWarnings({ "unchecked", "rawtypes" })
public WindowedDataStream<OUT> every(WindowingHelper... policyHelpers) {
WindowedDataStream<OUT> ret = this.copy();
@@ -137,11 +152,15 @@ public class WindowedDataStream<OUT> {
/**
* Groups the elements of the {@link WindowedDataStream} by the given key
- * positions to be used with grouped operators.
+ * positions. The window sizes (evictions) and slide sizes (triggers) will
+ * be calculated on the whole stream (in a central fashion), but the user
+ * defined functions will be applied on a per group basis. </br></br> To get
+ * windows and triggers on a per group basis apply the
+ * {@link DataStream#window} operator on an already grouped data stream.
*
* @param fields
* The position of the fields to group by.
- * @return The transformed {@link WindowedDataStream}
+ * @return The grouped {@link WindowedDataStream}
*/
public WindowedDataStream<OUT> groupBy(int... fields) {
WindowedDataStream<OUT> ret = this.copy();
@@ -153,11 +172,19 @@ public class WindowedDataStream<OUT> {
/**
* Groups the elements of the {@link WindowedDataStream} by the given field
- * expressions to be used with grouped operators.
+ * expressions. The window sizes (evictions) and slide sizes (triggers) will
+ * be calculated on the whole stream (in a central fashion), but the user
+ * defined functions will be applied on a per group basis. </br></br> To get
+ * windows and triggers on a per group basis apply the
+ * {@link DataStream#window} operator on an already grouped data stream.
+ * </br></br> A field expression is either the name of a public field or a
+ * getter method with parentheses of the stream's underlying type. A dot can
+ * be used to drill down into objects, as in
+ * {@code "field1.getInnerField2()" }.
*
* @param fields
- * The position of the fields to group by.
- * @return The transformed {@link WindowedDataStream}
+ * The fields to group by
+ * @return The grouped {@link WindowedDataStream}
*/
public WindowedDataStream<OUT> groupBy(String... fields) {
WindowedDataStream<OUT> ret = this.copy();
@@ -168,13 +195,16 @@ public class WindowedDataStream<OUT> {
}
/**
- * Groups the elements of the {@link WindowedDataStream} by the given
- * {@link KeySelector} to be used with grouped operators.
+ * Groups the elements of the {@link WindowedDataStream} using the given
+ * {@link KeySelector}. The window sizes (evictions) and slide sizes
+ * (triggers) will be calculated on the whole stream (in a central fashion),
+ * but the user defined functions will be applied on a per group basis.
+ * </br></br> To get windows and triggers on a per group basis apply the
+ * {@link DataStream#window} operator on an already grouped data stream.
*
* @param keySelector
- * The specification of the key on which the
- * {@link WindowedDataStream} will be grouped.
- * @return The transformed {@link WindowedDataStream}
+ * The keySelector used to extract the key for grouping.
+ * @return The grouped {@link WindowedDataStream}
*/
public WindowedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
WindowedDataStream<OUT> ret = this.copy();
@@ -185,17 +215,14 @@ public class WindowedDataStream<OUT> {
}
/**
- * This is a prototype implementation for new windowing features based on
- * trigger and eviction policies
+ * Applies a reduce transformation on the windowed data stream by reducing
+ * the current window at every trigger.The user can also extend the
+ * {@link RichReduceFunction} to gain access to other features provided by
+ * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
*
- * @param triggerHelpers
- * A list of trigger policies
- * @param evictionHelpers
- * A list of eviction policies
- * @param sample
- * A sample of the OUT data type required to gather type
- * information
- * @return The single output operator
+ * @param reduceFunction
+ * The reduce function that will be applied to the windows.
+ * @return The transformed DataStream
*/
public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reduceFunction) {
return dataStream.addFunction("NextGenWindowReduce", reduceFunction,
@@ -203,6 +230,19 @@ public class WindowedDataStream<OUT> {
getReduceInvokable(reduceFunction));
}
+ /**
+ * Applies a reduceGroup transformation on the windowed data stream by
+ * reducing the current window at every trigger. In contrast with the
+ * standard binary reducer, with reduceGroup the user can access all
+ * elements of the window at the same time through the iterable interface.
+ * The user can also extend the {@link RichGroupReduceFunction} to gain
+ * access to other features provided by the
+ * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+ *
+ * @param reduceFunction
+ * The reduce function that will be applied to the windows.
+ * @return The transformed DataStream
+ */
public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
GroupReduceFunction<OUT, R> reduceFunction) {
return dataStream.addFunction("NextGenWindowReduce", reduceFunction,
@@ -211,11 +251,11 @@ public class WindowedDataStream<OUT> {
}
/**
- * Applies an aggregation that sums every sliding batch/window of the data
- * stream at the given position.
+ * Applies an aggregation that sums every window of the data stream at the
+ * given position.
*
* @param positionToSum
- * The position in the data point to sum
+ * The position in the tuple/array to sum
* @return The transformed DataStream.
*/
public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
@@ -225,24 +265,14 @@ public class WindowedDataStream<OUT> {
}
/**
- * Syntactic sugar for sum(0)
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum() {
- return sum(0);
- }
-
- /**
- * Applies an aggregation that that gives the sum of the pojo data stream at
- * the given field expression. A field expression is either the name of a
- * public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
+ * Applies an aggregation that sums every window of the pojo data stream at
+ * the given field for every window. </br></br> A field expression is either
+ * the name of a public field or a getter method with parentheses of the
+ * stream's underlying type. A dot can be used to drill down into objects,
+ * as in {@code "field1.getInnerField2()" }.
*
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
+ * @param positionToSum
+ * The field to sum
* @return The transformed DataStream.
*/
public SingleOutputStreamOperator<OUT, ?> sum(String field) {
@@ -251,11 +281,11 @@ public class WindowedDataStream<OUT> {
}
/**
- * Applies an aggregation that that gives the minimum of every sliding
- * batch/window of the data stream at the given position.
+ * Applies an aggregation that that gives the minimum value of every window
+ * of the data stream at the given position.
*
* @param positionToMin
- * The position in the data point to minimize
+ * The position to minimize
* @return The transformed DataStream.
*/
public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
@@ -265,13 +295,29 @@ public class WindowedDataStream<OUT> {
}
/**
- * Applies an aggregation that gives the minimum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same minimum value the operator returns the first element by
- * default.
+ * Applies an aggregation that that gives the minimum value of the pojo data
+ * stream at the given field expression for every window. </br></br>A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min(String field) {
+ return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
+ AggregationType.MIN, false));
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * minimum value the operator returns the first element by default.
*
* @param positionToMinBy
- * The position in the data point to minimize
+ * The position to minimize by
* @return The transformed DataStream.
*/
public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
@@ -279,13 +325,13 @@ public class WindowedDataStream<OUT> {
}
/**
- * Applies an aggregation that gives the minimum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same minimum value the operator returns either the first or last
- * one depending on the parameter setting.
+ * Applies an aggregation that gives the minimum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * minimum value the operator returns either the first or last one depending
+ * on the parameter setting.
*
* @param positionToMinBy
- * The position in the data point to minimize
+ * The position to minimize
* @param first
* If true, then the operator return the first element with the
* minimum value, otherwise returns the last
@@ -298,20 +344,31 @@ public class WindowedDataStream<OUT> {
}
/**
- * Syntactic sugar for min(0)
+ * Applies an aggregation that that gives the minimum element of the pojo
+ * data stream by the given field expression for every window. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
*
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @param first
+ * If True then in case of field equality the first object will
+ * be returned
* @return The transformed DataStream.
*/
- public SingleOutputStreamOperator<OUT, ?> min() {
- return min(0);
+ public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
+ return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
+ AggregationType.MINBY, first));
}
/**
- * Applies an aggregation that gives the maximum of every sliding
- * batch/window of the data stream at the given position.
+ * Applies an aggregation that gives the maximum value of every window of
+ * the data stream at the given position.
*
* @param positionToMax
- * The position in the data point to maximize
+ * The position to maximize
* @return The transformed DataStream.
*/
public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
@@ -321,106 +378,60 @@ public class WindowedDataStream<OUT> {
}
/**
- * Applies an aggregation that gives the maximum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same maximum value the operator returns the first by default.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
- return this.maxBy(positionToMaxBy, true);
- }
-
- /**
- * Applies an aggregation that gives the maximum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same maximum value the operator returns either the first or last
- * one depending on the parameter setting.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @param first
- * If true, then the operator return the first element with the
- * maximum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
- dataStream.checkFieldRange(positionToMaxBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getOutputType(),
- AggregationType.MAXBY, first));
- }
-
- /**
- * Syntactic sugar for max(0)
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max() {
- return max(0);
- }
-
- /**
- * Applies an aggregation that that gives the minimum of the pojo data
- * stream at the given field expression. A field expression is either the
- * name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
+ * Applies an aggregation that that gives the maximum value of the pojo data
+ * stream at the given field expression for every window. A field expression
+ * is either the name of a public field or a getter method with parentheses
+ * of the {@link DataStream}S underlying type. A dot can be used to drill
+ * down into objects, as in {@code "field1.getInnerField2()" }.
*
* @param field
* The field expression based on which the aggregation will be
* applied.
* @return The transformed DataStream.
*/
- public SingleOutputStreamOperator<OUT, ?> min(String field) {
+ public SingleOutputStreamOperator<OUT, ?> max(String field) {
return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
- AggregationType.MIN, false));
+ AggregationType.MAX, false));
}
/**
- * Applies an aggregation that that gives the maximum of the pojo data
- * stream at the given field expression. A field expression is either the
- * name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
+ * Applies an aggregation that gives the maximum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * maximum value the operator returns the first by default.
*
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
+ * @param positionToMaxBy
+ * The position to maximize by
* @return The transformed DataStream.
*/
- public SingleOutputStreamOperator<OUT, ?> max(String field) {
- return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
- AggregationType.MAX, false));
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
}
/**
- * Applies an aggregation that that gives the minimum element of the pojo
- * data stream by the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
+ * Applies an aggregation that gives the maximum element of every window of
+ * the data stream by the given position. If more elements have the same
+ * maximum value the operator returns either the first or last one depending
+ * on the parameter setting.
*
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
+ * @param positionToMaxBy
+ * The position to maximize by
* @param first
- * If True then in case of field equality the first object will
- * be returned
+ * If true, then the operator return the first element with the
+ * maximum value, otherwise returns the last
* @return The transformed DataStream.
*/
- public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
- return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
- AggregationType.MINBY, first));
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
+ dataStream.checkFieldRange(positionToMaxBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getOutputType(),
+ AggregationType.MAXBY, first));
}
/**
* Applies an aggregation that that gives the maximum element of the pojo
- * data stream by the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
+ * data stream by the given field expression for every window. A field
+ * expression is either the name of a public field or a getter method with
+ * parentheses of the {@link DataStream}S underlying type. A dot can be used
+ * to drill down into objects, as in {@code "field1.getInnerField2()" }.
*
* @param field
* The field expression based on which the aggregation will be
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
index 73f7daf..840546f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
@@ -85,8 +85,10 @@ public class Count implements WindowingHelper {
}
/**
- * Specifies on which element a trigger or an eviction should happen (based
- * on the count of the elements)
+ * Specifies a count based eviction (window size) or trigger policy (slide
+ * size). For eviction 'count' defines the number of elements in each
+ * window. For trigger 'count' defines how often do we call the user
+ * function in terms of number of elements received.
*
* @param count
* the number of elements to count before trigger/evict
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
index ce75fc5..9c8c5ca 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Delta.java
@@ -65,7 +65,14 @@ public class Delta<DATA> implements WindowingHelper<DATA> {
}
/**
- * Creates a delta helper representing a delta count or eviction policy
+ * Creates a delta helper representing a delta trigger or eviction policy.
+ * </br></br> This policy calculates a delta between the data point which
+ * triggered last and the currently arrived data point. It triggers if the
+ * delta is higher than a specified threshold. </br></br> In case it gets
+ * used for eviction, this policy starts from the first element of the
+ * buffer and removes all elements from the buffer which have a higher delta
+ * then the threshold. As soon as there is an element with a lower delta,
+ * the eviction stops.
*
* @param deltaFunction
* The delta function which should be used to calculate the delta
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
index 908d0cf..d987e32 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -75,8 +75,9 @@ public class Time<DATA> implements WindowingHelper<DATA> {
}
/**
- * Creates an helper representing a trigger which triggers every given
- * length or an eviction which evicts all elements older than length.
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size).
*
* @param length
* The number of time units
@@ -92,8 +93,9 @@ public class Time<DATA> implements WindowingHelper<DATA> {
}
/**
- * Creates an helper representing a trigger which triggers every given
- * length or an eviction which evicts all elements older than length.
+ * Creates a helper representing a time trigger which triggers every given
+ * length (slide size) or a time eviction which evicts all elements older
+ * than length (window size) using a user defined timestamp extractor.
*
* @param length
* The number of time units
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/4e046a9b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
index 128d3bf..8c26e4a 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
@@ -64,7 +64,7 @@ public class TimeWindowingExample {
.window(Count.of(100))
.every(Time.of(1000, TimeUnit.MILLISECONDS))
.groupBy(myKey)
- .sum();
+ .sum(0);
stream.print();
[10/34] incubator-flink git commit: [streaming] Added abstract class
ExtractionAwareDeltaFunction to enable easy extraction of data in pre-defined
delta functions.
Posted by mb...@apache.org.
[streaming] Added abstract class ExtractionAwareDeltaFunction to enable easy extraction of data in pre-defined delta functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/b2636247
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/b2636247
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/b2636247
Branch: refs/heads/master
Commit: b2636247616fe152f85b3afeb03da3fa15689b09
Parents: 997d3e1
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 14:02:21 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../ExtractionAwareDeltaFunction.java | 90 ++++++++++++++++++++
1 file changed, 90 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/b2636247/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
new file mode 100644
index 0000000..81d01a4
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/ExtractionAwareDeltaFunction.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * Extend this abstract class to implement a delta function which is aware of
+ * extracting the data on which the delta is calculated from a more complex data
+ * structure. For example in case you want to be able to run a delta only on one
+ * field of a Tuple type or only on some fields from an array.
+ *
+ * @param <DATA>
+ * The input data type. The input of this type will be passed to the
+ * extractor which will transform into a TO-object. The delta
+ * function then runs on this TO-object.
+ * @param <TO>
+ * The type on which the delta function runs. (The type of the delta
+ * function)
+ */
+public abstract class ExtractionAwareDeltaFunction<DATA, TO> implements DeltaFunction<DATA> {
+
+ /**
+ * Generated Version ID
+ */
+ private static final long serialVersionUID = 6927486219702689554L;
+ private Extractor<DATA, TO> converter;
+
+ public ExtractionAwareDeltaFunction(Extractor<DATA, TO> converter) {
+ this.converter = converter;
+ }
+
+ /**
+ * This method takes the two data point and runs the set extractor on it.
+ * The delta function implemented at {@link getNestedDelta} is then called
+ * with the extracted data. In case no extractor is set the input data gets
+ * passes to {@link getNestedDelta} as-is. The return value is just
+ * forwarded from {@link getNestedDelta}.
+ *
+ * @param oldDataPoint
+ * the older data point as raw data (before extraction).
+ * @param newDataPoint
+ * the new data point as raw data (before extraction).
+ * @return the delta between the two points.
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public double getDelta(DATA oldDataPoint, DATA newDataPoint) {
+ if (converter == null) {
+ // In case no conversion/extraction is required, we can cast DATA to
+ // TO
+ // => Therefore, "unchecked" warning is suppressed for this method.
+ return getNestedDelta((TO) oldDataPoint, (TO) newDataPoint);
+ } else {
+ return getNestedDelta(converter.extract(oldDataPoint), converter.extract(newDataPoint));
+ }
+
+ }
+
+ /**
+ * This method is exactly the same as
+ * {@link DeltaFunction#getDelta(Object, Object)} except that it gets the
+ * result of the previously done extractions as input. Therefore, this
+ * method only does the actual calculation of the delta but no data
+ * extraction or conversion.
+ *
+ * @param oldDataPoint
+ * the older data point.
+ * @param newDataPoint
+ * the new data point.
+ * @return the delta between the two points.
+ */
+ public abstract double getNestedDelta(TO oldDataPoint, TO newDataPoint);
+
+}
[04/34] incubator-flink git commit: [streaming] Added interface for
policy helpers. Such helpers will allow to use policy based windowing through
a simpler API.
Posted by mb...@apache.org.
[streaming] Added interface for policy helpers. Such helpers will allow to use policy based windowing through a simpler API.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/93178aeb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/93178aeb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/93178aeb
Branch: refs/heads/master
Commit: 93178aebb804e2e96a4e6ae7e016247a4e2d82c7
Parents: 36f1ee8
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:16:33 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../api/windowing/helper/WindowingHelper.java | 39 ++++++++++++++++++++
1 file changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/93178aeb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
new file mode 100644
index 0000000..9df8432
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/WindowingHelper.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * A helper representing a count or eviction policy. Such helper classes are
+ * used to provide a nice and well readable API.
+ *
+ * @param <DATA>
+ * the type of input data handled by this helper
+ * @see Count
+ * @see Time
+ * @see Delta
+ */
+public interface WindowingHelper<DATA> {
+
+ public EvictionPolicy<DATA> toEvict();
+
+ public TriggerPolicy<DATA> toTrigger();
+
+}
[19/34] incubator-flink git commit: [streaming] Extractor requirement
removed for custom timestamp windowing
Posted by mb...@apache.org.
[streaming] Extractor requirement removed for custom timestamp windowing
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/55ee64f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/55ee64f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/55ee64f6
Branch: refs/heads/master
Commit: 55ee64f6c9824d51922fe4a8fb1435030925b110
Parents: f96ba06
Author: Gyula Fora <gy...@apache.org>
Authored: Mon Nov 24 00:32:52 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../operator/GroupedWindowingInvokable.java | 11 +++--
.../invokable/operator/WindowingInvokable.java | 10 ++---
.../streaming/api/windowing/helper/Time.java | 8 +---
.../windowing/policy/ActiveEvictionPolicy.java | 2 +-
.../policy/ActiveEvictionPolicyWrapper.java | 5 ++-
.../windowing/policy/ActiveTriggerCallback.java | 11 ++---
.../windowing/policy/ActiveTriggerPolicy.java | 16 +++----
.../windowing/policy/TimeEvictionPolicy.java | 46 ++++++++++++++------
.../api/windowing/policy/TimeTriggerPolicy.java | 42 ++++++++----------
.../apache/flink/streaming/api/PrintTest.java | 8 ----
.../operator/GroupedWindowingInvokableTest.java | 13 +-----
.../operator/WindowingInvokableTest.java | 13 +-----
.../windowing/policy/TimeTriggerPolicyTest.java | 16 ++-----
13 files changed, 84 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
index e957b8c..c341e56 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
@@ -138,8 +138,7 @@ public class GroupedWindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
* If only one element is contained a group, this element itself
* is returned as aggregated result.)
*/
- public GroupedWindowingInvokable(Function userFunction,
- KeySelector<IN, ?> keySelector,
+ public GroupedWindowingInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
LinkedList<TriggerPolicy<IN>> centralTriggerPolicies) {
@@ -174,8 +173,8 @@ public class GroupedWindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
// Run the precalls for central active triggers
for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
- IN[] result = trigger.preNotifyTrigger(reuse.getObject());
- for (IN in : result) {
+ Object[] result = trigger.preNotifyTrigger(reuse.getObject());
+ for (Object in : result) {
for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(in, trigger);
}
@@ -305,7 +304,7 @@ public class GroupedWindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
*
* @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
*/
- private class WindowingCallback implements ActiveTriggerCallback<IN> {
+ private class WindowingCallback implements ActiveTriggerCallback {
private ActiveTriggerPolicy<IN> policy;
public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
@@ -313,7 +312,7 @@ public class GroupedWindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT>
}
@Override
- public void sendFakeElement(IN datapoint) {
+ public void sendFakeElement(Object datapoint) {
for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(datapoint, policy);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
index d0855e6..95a999c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
@@ -101,7 +101,7 @@ public abstract class WindowingInvokable<IN, OUT> extends StreamInvokable<IN, OU
* This class allows the active trigger threads to call back and push fake
* elements at any time.
*/
- private class WindowingCallback implements ActiveTriggerCallback<IN> {
+ private class WindowingCallback implements ActiveTriggerCallback {
private ActiveTriggerPolicy<IN> policy;
public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
@@ -109,7 +109,7 @@ public abstract class WindowingInvokable<IN, OUT> extends StreamInvokable<IN, OU
}
@Override
- public void sendFakeElement(IN datapoint) {
+ public void sendFakeElement(Object datapoint) {
processFakeElement(datapoint, this.policy);
}
@@ -215,7 +215,7 @@ public abstract class WindowingInvokable<IN, OUT> extends StreamInvokable<IN, OU
* @param currentPolicy
* the policy which produced this fake element
*/
- protected synchronized void processFakeElement(IN input, TriggerPolicy<IN> currentPolicy) {
+ protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
// Process the evictions and take care of double evictions
// In case there are multiple eviction policies present,
@@ -283,8 +283,8 @@ public abstract class WindowingInvokable<IN, OUT> extends StreamInvokable<IN, OU
// of the different fake elements returned by this triggers becomes
// a problem. This might lead to unexpected results...
// Should we limit the number of active triggers to 0 or 1?
- IN[] result = trigger.preNotifyTrigger(input);
- for (IN in : result) {
+ Object[] result = trigger.preNotifyTrigger(input);
+ for (Object in : result) {
processFakeElement(in, trigger);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
index 0b73150..86ad952 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -40,7 +40,6 @@ public class Time<DATA> implements WindowingHelper<DATA> {
private int timeVal;
private TimeUnit granularity;
- private Extractor<Long, DATA> longToDATAExtractor;
private TimeStamp<DATA> timeStamp;
private long delay;
@@ -59,7 +58,6 @@ public class Time<DATA> implements WindowingHelper<DATA> {
private Time(int timeVal, TimeUnit timeUnit) {
this.timeVal = timeVal;
this.granularity = timeUnit;
- this.longToDATAExtractor = new NullExtractor<DATA>();
this.timeStamp = new DefaultTimeStamp<DATA>();
this.delay = 0;
}
@@ -71,8 +69,7 @@ public class Time<DATA> implements WindowingHelper<DATA> {
@Override
public TriggerPolicy<DATA> toTrigger() {
- return new TimeTriggerPolicy<DATA>(granularityInMillis(), timeStamp, delay,
- longToDATAExtractor);
+ return new TimeTriggerPolicy<DATA>(granularityInMillis(), timeStamp, delay);
}
/**
@@ -95,9 +92,8 @@ public class Time<DATA> implements WindowingHelper<DATA> {
}
@SuppressWarnings("unchecked")
- public <R> Time<R> withTimeStamp(TimeStamp<R> timeStamp, Extractor<Long, R> extractor) {
+ public <R> Time<R> withTimeStamp(TimeStamp<R> timeStamp) {
this.timeStamp = (TimeStamp<DATA>) timeStamp;
- this.longToDATAExtractor = (Extractor<Long, DATA>) extractor;
return (Time<R>) this;
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
index 17d4914..fe172bc 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicy.java
@@ -47,6 +47,6 @@ public interface ActiveEvictionPolicy<DATA> extends EvictionPolicy<DATA> {
* @return the number of elements to delete from the buffer (only real
* elements are counted)
*/
- public int notifyEvictionWithFakeElement(DATA datapoint, int bufferSize);
+ public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
index a110fbc..b3b6935 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveEvictionPolicyWrapper.java
@@ -55,9 +55,10 @@ public class ActiveEvictionPolicyWrapper<DATA> implements ActiveEvictionPolicy<D
return nestedPolicy.notifyEviction(datapoint, triggered, bufferSize);
}
+ @SuppressWarnings("unchecked")
@Override
- public int notifyEvictionWithFakeElement(DATA datapoint, int bufferSize) {
- return nestedPolicy.notifyEviction(datapoint, true, bufferSize);
+ public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize) {
+ return nestedPolicy.notifyEviction((DATA) datapoint, true, bufferSize);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
index d74a3ae..c44be37 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerCallback.java
@@ -20,18 +20,15 @@ package org.apache.flink.streaming.api.windowing.policy;
/**
* In case an {@link ActiveTriggerPolicy} is used, it can implement own
* {@link Runnable} classes. Such {@link Runnable} classes will be executed as
- * an own thread and can submit fake elements, to the element
- * buffer at any time.
+ * an own thread and can submit fake elements, to the element buffer at any
+ * time.
*
* The factory method for runnables of the {@link ActiveTriggerPolicy} gets an
* instance of this interface as parameter. The describes adding of elements can
* be done by the runnable using the methods provided in this interface.
*
- * @param <DATA>
- * The data type which can be consumed by the methods provided in
- * this callback implementation.
*/
-public interface ActiveTriggerCallback<DATA> {
+public interface ActiveTriggerCallback {
/**
* Submits a new fake data point to the element buffer. Such a fake element
@@ -43,6 +40,6 @@ public interface ActiveTriggerCallback<DATA> {
* @param datapoint
* the fake data point to be added
*/
- public void sendFakeElement(DATA datapoint);
+ public void sendFakeElement(Object datapoint);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
index f549766..a8a704d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/ActiveTriggerPolicy.java
@@ -24,12 +24,12 @@ import org.apache.flink.streaming.api.invokable.util.TimeStamp;
* for active triggers. Active triggers can act in two ways:
*
* 1) Whenever an element arrives at the invokable, the
- * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called first.
- * It can return zero ore more fake data points which will be added before the
- * the currently arrived real element gets processed. This allows to handle
- * empty windows in time based windowing with an user defined {@link TimeStamp}.
- * Triggers are not called on fake datapoint. A fake datapoint is always
- * considered as triggered.
+ * {@link ActiveTriggerPolicy#preNotifyTrigger(Object)} method gets called
+ * first. It can return zero ore more fake data points which will be added
+ * before the the currently arrived real element gets processed. This allows to
+ * handle empty windows in time based windowing with an user defined
+ * {@link TimeStamp}. Triggers are not called on fake datapoint. A fake
+ * datapoint is always considered as triggered.
*
* 2) An active trigger has a factory method for a runnable. This factory method
* gets called at the start up of the invokable. The returned runnable will be
@@ -57,7 +57,7 @@ public interface ActiveTriggerPolicy<DATA> extends TriggerPolicy<DATA> {
* @return zero ore more fake data points which will be added before the the
* currently arrived real element gets processed.
*/
- public DATA[] preNotifyTrigger(DATA datapoint);
+ public Object[] preNotifyTrigger(DATA datapoint);
/**
* This is the factory method for a runnable. This factory method gets
@@ -76,6 +76,6 @@ public interface ActiveTriggerPolicy<DATA> extends TriggerPolicy<DATA> {
* executed as an own thread and can submit fake elements, to the
* element buffer at any time.
*/
- public Runnable createActiveTriggerRunnable(ActiveTriggerCallback<DATA> callback);
+ public Runnable createActiveTriggerRunnable(ActiveTriggerCallback callback);
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
index b212293..e886c09 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
@@ -69,25 +69,49 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
this.granularity = granularity;
}
+ @SuppressWarnings("unchecked")
@Override
- public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
- return notifyEviction(datapoint, triggered, bufferSize, false);
+ public int notifyEvictionWithFakeElement(Object datapoint, int bufferSize) {
+ checkForDeleted(bufferSize);
+
+ long threshold;
+ try {
+ threshold = (Long) datapoint - granularity;
+ } catch (ClassCastException e) {
+ threshold = timestamp.getTimestamp((DATA) datapoint) - granularity;
+ }
+
+ // return result
+ return deleteAndCountExpired(threshold);
+
}
@Override
- public int notifyEvictionWithFakeElement(DATA datapoint, int bufferSize) {
- return notifyEviction(datapoint, true, bufferSize, true);
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
+
+ checkForDeleted(bufferSize);
+
+ // delete and count expired tuples
+ long threshold = timestamp.getTimestamp(datapoint) - granularity;
+ int counter = deleteAndCountExpired(threshold);
+
+ // Add current element to buffer
+ buffer.add(datapoint);
+
+ // return result
+ return counter;
+
}
- private int notifyEviction(DATA datapoint, boolean triggered, int bufferSize, boolean isFake) {
+ private void checkForDeleted(int bufferSize) {
// check for deleted tuples (deletes by other policies)
while (bufferSize < this.buffer.size()) {
this.buffer.removeFirst();
}
+ }
- // delete and count expired tuples
+ private int deleteAndCountExpired(long threshold) {
int counter = 0;
- long threshold = timestamp.getTimestamp(datapoint) - granularity;
while (!buffer.isEmpty()) {
if (timestamp.getTimestamp(buffer.getFirst()) < threshold) {
@@ -97,14 +121,8 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
break;
}
}
-
- if (!isFake) {
- // Add current element to buffer
- buffer.add(datapoint);
- }
-
- // return result
return counter;
+
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
index 1dd713f..c3402c9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -21,7 +21,6 @@ import java.util.LinkedList;
import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
/**
* This trigger policy triggers with regard to the time. The is measured using a
@@ -46,8 +45,6 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
protected TimeStamp<DATA> timestamp;
protected long delay;
- private Extractor<Long, DATA> longToDATAExtractor;
-
/**
* This trigger policy triggers with regard to the time. The is measured
* using a given {@link TimeStamp} implementation. A point in time is always
@@ -67,17 +64,16 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* extractor should wrap a long into such an element of type
* DATA.
*/
- public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp,
- Extractor<Long, DATA> timeWrapper) {
- this(granularity, timestamp, 0, timeWrapper);
+ public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp) {
+ this(granularity, timestamp, 0);
}
/**
* This is mostly the same as
- * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, TimeStamp)}. In
- * addition to granularity and timestamp a delay can be specified for the
- * first trigger. If the start time given by the timestamp is x, the delay
- * is y, and the granularity is z, the first trigger will happen at x+y+z.
+ * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, TimeStamp)}. In addition
+ * to granularity and timestamp a delay can be specified for the first
+ * trigger. If the start time given by the timestamp is x, the delay is y,
+ * and the granularity is z, the first trigger will happen at x+y+z.
*
* @param granularity
* The granularity of the trigger. If this value is set to 2 the
@@ -95,31 +91,27 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* extractor should wrap a long into such an element of type
* DATA.
*/
- public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp, long delay,
- Extractor<Long, DATA> timeWrapper) {
+ public TimeTriggerPolicy(long granularity, TimeStamp<DATA> timestamp, long delay) {
this.startTime = timestamp.getStartTime() + delay;
this.timestamp = timestamp;
this.granularity = granularity;
this.delay = delay;
- this.longToDATAExtractor = timeWrapper;
-
}
/**
* This method checks if we missed a window end. If this is the case we
* trigger the missed windows using fake elements.
*/
- @SuppressWarnings("unchecked")
@Override
- public synchronized DATA[] preNotifyTrigger(DATA datapoint) {
- LinkedList<DATA> fakeElements = new LinkedList<DATA>();
+ public synchronized Object[] preNotifyTrigger(DATA datapoint) {
+ LinkedList<Object> fakeElements = new LinkedList<Object>();
// check if there is more then one window border missed
// use > here. In case >= would fit, the regular call will do the job.
while (timestamp.getTimestamp(datapoint) > startTime + granularity) {
startTime += granularity;
- fakeElements.add(longToDATAExtractor.extract(startTime));
+ fakeElements.add(startTime);
}
- return (DATA[]) fakeElements.toArray();
+ return (Object[]) fakeElements.toArray();
}
/**
@@ -134,7 +126,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* time. If any other time measure is used the method return null.
*/
@Override
- public Runnable createActiveTriggerRunnable(ActiveTriggerCallback<DATA> callback) {
+ public Runnable createActiveTriggerRunnable(ActiveTriggerCallback callback) {
if (this.timestamp instanceof DefaultTimeStamp) {
return new TimeCheck(callback);
} else {
@@ -149,19 +141,19 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* @param callback
* The callback object.
*/
- private synchronized void activeFakeElementEmission(ActiveTriggerCallback<DATA> callback) {
+ private synchronized void activeFakeElementEmission(ActiveTriggerCallback callback) {
if (System.currentTimeMillis() >= startTime + granularity) {
startTime += granularity;
- callback.sendFakeElement(longToDATAExtractor.extract(startTime));
+ callback.sendFakeElement(startTime);
}
}
private class TimeCheck implements Runnable {
- ActiveTriggerCallback<DATA> callback;
+ ActiveTriggerCallback callback;
- public TimeCheck(ActiveTriggerCallback<DATA> callback) {
+ public TimeCheck(ActiveTriggerCallback callback) {
this.callback = callback;
}
@@ -198,7 +190,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
@Override
public TimeTriggerPolicy<DATA> clone() {
- return new TimeTriggerPolicy<DATA>(granularity, timestamp, delay, longToDATAExtractor);
+ return new TimeTriggerPolicy<DATA>(granularity, timestamp, delay);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
index c494d5f..f7ba82e 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
@@ -81,14 +81,6 @@ public class PrintTest implements Serializable {
public long getStartTime() {
return 1;
}
- }, new Extractor<Long, Integer>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer extract(Long in) {
- return in.intValue();
- }
})).every(Count.of(2)).reduceGroup(new GroupReduceFunction<Integer, String>() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
index ebfed05..79288d1 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
@@ -28,14 +28,13 @@ import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
import org.apache.flink.streaming.util.MockInvokable;
@@ -249,15 +248,7 @@ public class GroupedWindowingInvokableTest {
LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
// Trigger every 2 time units but delay the first trigger by 2 (First
// trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L,
- new Extractor<Long, Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, String> extract(Long in) {
- return new Tuple2<Integer, String>(in.intValue(), null);
- }
- }));
+ triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStamp, 2L));
LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
// Always delete all elements older then 4
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
index 8e96a7c..32e71ba 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
@@ -25,12 +25,11 @@ import java.util.List;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.util.MockInvokable;
import org.junit.Test;
@@ -89,16 +88,8 @@ public class WindowingInvokableTest {
LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
// Trigger every 2 time units but delay the first trigger by 2 (First
// trigger after 4, then every 2)
- triggers.add(new TimeTriggerPolicy<Integer>(2L, myTimeStamp, 2L,
- new Extractor<Long, Integer>() {
-
- private static final long serialVersionUID = 1L;
+ triggers.add(new TimeTriggerPolicy<Integer>(2L, myTimeStamp, 2L));
- @Override
- public Integer extract(Long in) {
- return in.intValue();
- }
- }));
LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
// Always delete all elements older then 4
evictions.add(new TimeEvictionPolicy<Integer>(4L, myTimeStamp));
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/55ee64f6/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
index 34e292a..1355b27 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
@@ -22,8 +22,6 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
-import org.apache.flink.streaming.api.windowing.helper.Time;
import org.junit.Test;
public class TimeTriggerPolicyTest {
@@ -53,7 +51,7 @@ public class TimeTriggerPolicyTest {
for (long granularity = 0; granularity < 31; granularity++) {
// create policy
TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity,
- timeStamp, new Time.NullExtractor<Integer>());
+ timeStamp);
// remember window border
// Remark: This might NOT work in case the timeStamp uses
@@ -104,18 +102,10 @@ public class TimeTriggerPolicyTest {
// create policy
TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5,
- timeStamp, new Extractor<Long, Integer>() {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer extract(Long in) {
- return in.intValue();
- }
- });
+ timeStamp);
// expected result
- Integer[][] result = { {}, {}, { 5, 10, 15 }, { 25 } };
+ Long[][] result = { {}, {}, { 5L, 10L, 15L }, { 25L } };
// call policy
for (int i = 0; i < times.length; i++) {
[15/34] incubator-flink git commit: [streaming] New windowing API
merge and cleanup + several minor fixes
Posted by mb...@apache.org.
[streaming] New windowing API merge and cleanup + several minor fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/f96ba06e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/f96ba06e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/f96ba06e
Branch: refs/heads/master
Commit: f96ba06e0510690af8c44ae5ef09dda15d2e1ff4
Parents: 751f101
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Nov 23 18:54:44 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../api/datastream/BatchedDataStream.java | 414 -------------------
.../streaming/api/datastream/DataStream.java | 96 -----
.../api/datastream/WindowDataStream.java | 92 -----
.../api/datastream/WindowedDataStream.java | 403 ++++++++++++++++--
.../operator/BatchGroupReduceInvokable.java | 163 --------
.../operator/BatchReduceInvokable.java | 224 ----------
.../GroupedBatchGroupReduceInvokable.java | 61 ---
.../operator/GroupedBatchReduceInvokable.java | 58 ---
.../GroupedWindowGroupReduceInvokable.java | 128 ------
.../operator/GroupedWindowReduceInvokable.java | 106 -----
.../operator/GroupedWindowingInvokable.java | 41 +-
.../operator/WindowGroupReduceInvokable.java | 108 -----
.../operator/WindowReduceInvokable.java | 124 ------
.../operator/WindowingGroupInvokable.java | 43 ++
.../invokable/operator/WindowingInvokable.java | 44 +-
.../operator/WindowingReduceInvokable.java | 59 +++
.../streaming/api/windowing/helper/Count.java | 25 +-
.../streaming/api/windowing/helper/Time.java | 79 ++--
.../windowing/policy/CountTriggerPolicy.java | 2 +-
.../windowing/policy/TimeEvictionPolicy.java | 4 +-
.../api/windowing/policy/TimeTriggerPolicy.java | 55 +--
.../apache/flink/streaming/api/PrintTest.java | 60 ++-
.../operator/BatchGroupReduceTest.java | 98 -----
.../api/invokable/operator/BatchReduceTest.java | 85 ----
.../operator/GroupedBatchGroupReduceTest.java | 101 -----
.../operator/GroupedBatchReduceTest.java | 107 -----
.../GroupedWindowGroupReduceInvokableTest.java | 99 -----
.../operator/GroupedWindowingInvokableTest.java | 44 +-
.../WindowGroupReduceInvokableTest.java | 124 ------
.../operator/WindowReduceInvokableTest.java | 130 ------
.../operator/WindowingInvokableTest.java | 42 +-
.../windowing/policy/TimeTriggerPolicyTest.java | 39 +-
.../ml/IncrementalLearningSkeleton.java | 6 +-
.../examples/windowing/BasicExample.java | 76 ----
.../windowing/MultiplePoliciesExample.java | 38 +-
.../examples/windowing/SlidingExample.java | 27 +-
.../windowing/TimeWindowingExample.java | 57 +--
37 files changed, 771 insertions(+), 2691 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/BatchedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/BatchedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/BatchedDataStream.java
deleted file mode 100755
index c8a49c6..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/BatchedDataStream.java
+++ /dev/null
@@ -1,414 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.common.functions.RichReduceFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
-import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
-import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
-import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.BatchGroupReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.BatchReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.GroupedBatchGroupReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.GroupedBatchReduceInvokable;
-import org.apache.flink.streaming.util.serialization.FunctionTypeWrapper;
-
-/**
- * A {@link BatchedDataStream} represents a data stream whose elements are
- * batched together in a sliding batch. operations like
- * {@link #reduce(ReduceFunction)} or {@link #reduceGroup(GroupReduceFunction)}
- * are applied for each batch and the batch is slid afterwards.
- *
- * @param <OUT>
- * The output type of the {@link BatchedDataStream}
- */
-public class BatchedDataStream<OUT> {
-
- protected DataStream<OUT> dataStream;
- protected boolean isGrouped;
- protected KeySelector<OUT, ?> keySelector;
- protected long batchSize;
- protected long slideSize;
-
- protected BatchedDataStream(DataStream<OUT> dataStream, long batchSize, long slideSize) {
- if (dataStream instanceof GroupedDataStream) {
- this.isGrouped = true;
- this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
- } else {
- this.isGrouped = false;
- }
- this.dataStream = dataStream.copy();
- this.batchSize = batchSize;
- this.slideSize = slideSize;
- }
-
- protected BatchedDataStream(BatchedDataStream<OUT> batchedDataStream) {
- this.dataStream = batchedDataStream.dataStream.copy();
- this.isGrouped = batchedDataStream.isGrouped;
- this.keySelector = batchedDataStream.keySelector;
- this.batchSize = batchedDataStream.batchSize;
- this.slideSize = batchedDataStream.slideSize;
- }
-
- /**
- * Groups the elements of the {@link BatchedDataStream} by the given key
- * positions to be used with grouped operators.
- *
- * @param fields
- * The position of the fields on which the
- * {@link BatchedDataStream} will be grouped.
- * @return The transformed {@link BatchedDataStream}
- */
- public BatchedDataStream<OUT> groupBy(int... fields) {
- return new BatchedDataStream<OUT>(dataStream.groupBy(fields), batchSize, slideSize);
- }
-
- /**
- * Groups a {@link BatchedDataStream} using field expressions. A field
- * expression is either the name of a public field or a getter method with
- * parentheses of the {@link BatchedDataStream}S underlying type. A dot can
- * be used to drill down into objects, as in
- * {@code "field1.getInnerField2()" }.
- *
- * @param fields
- * One or more field expressions on which the DataStream will be
- * grouped.
- * @return The grouped {@link BatchedDataStream}
- **/
- public BatchedDataStream<OUT> groupBy(String... fields) {
-
- return new BatchedDataStream<OUT>(dataStream.groupBy(fields), batchSize, slideSize);
-
- }
-
- /**
- * Applies a reduce transformation on every sliding batch/window of the data
- * stream. If the data stream is grouped then the reducer is applied on
- * every group of elements sharing the same key. This type of reduce is much
- * faster than reduceGroup since the reduce function can be applied
- * incrementally. The user can also extend the {@link RichReduceFunction} to
- * gain access to other features provided by the {@link RichFuntion}
- * interface.
- *
- * @param reducer
- * The {@link ReduceFunction} that will be called for every
- * element of the input values in the batch/window.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reducer) {
- return dataStream.addFunction("batchReduce", reducer, new FunctionTypeWrapper<OUT>(reducer,
- ReduceFunction.class, 0), new FunctionTypeWrapper<OUT>(reducer,
- ReduceFunction.class, 0), getReduceInvokable(reducer));
- }
-
- /**
- * Applies a reduceGroup transformation on preset batches/windows of the
- * DataStream. The transformation calls a {@link GroupReduceFunction} for
- * each batch/window. Each GroupReduceFunction call can return any number of
- * elements including none. The user can also extend
- * {@link RichGroupReduceFunction} to gain access to other features provided
- * by the {@link RichFuntion} interface.
- *
- * @param reducer
- * The {@link GroupReduceFunction} that will be called for every
- * batch/window.
- * @return The transformed DataStream.
- */
- public <R> SingleOutputStreamOperator<R, ?> reduceGroup(GroupReduceFunction<OUT, R> reducer) {
- return dataStream.addFunction("batchReduce", reducer, new FunctionTypeWrapper<OUT>(reducer,
- GroupReduceFunction.class, 0), new FunctionTypeWrapper<R>(reducer,
- GroupReduceFunction.class, 1), getGroupReduceInvokable(reducer));
- }
-
- /**
- * Applies an aggregation that sums every sliding batch/window of the data
- * stream at the given position.
- *
- * @param positionToSum
- * The position in the data point to sum
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
- dataStream.checkFieldRange(positionToSum);
- return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(positionToSum,
- dataStream.getClassAtPos(positionToSum), dataStream.getOutputType()));
- }
-
- /**
- * Syntactic sugar for sum(0)
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum() {
- return sum(0);
- }
-
- /**
- * Applies an aggregation that that gives the sum of the pojo data stream at
- * the given field expression. A field expression is either the name of a
- * public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> sum(String field) {
- return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field,
- getOutputType()));
- }
-
- /**
- * Applies an aggregation that that gives the minimum of every sliding
- * batch/window of the data stream at the given position.
- *
- * @param positionToMin
- * The position in the data point to minimize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
- dataStream.checkFieldRange(positionToMin);
- return aggregate(ComparableAggregator.getAggregator(positionToMin, getOutputType(),
- AggregationType.MIN));
- }
-
- /**
- * Applies an aggregation that gives the minimum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same minimum value the operator returns the first element by
- * default.
- *
- * @param positionToMinBy
- * The position in the data point to minimize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
- return this.minBy(positionToMinBy, true);
- }
-
- /**
- * Applies an aggregation that gives the minimum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same minimum value the operator returns either the first or last
- * one depending on the parameter setting.
- *
- * @param positionToMinBy
- * The position in the data point to minimize
- * @param first
- * If true, then the operator return the first element with the
- * minimum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
- dataStream.checkFieldRange(positionToMinBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMinBy, getOutputType(),
- AggregationType.MINBY, first));
- }
-
- /**
- * Syntactic sugar for min(0)
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min() {
- return min(0);
- }
-
- /**
- * Applies an aggregation that gives the maximum of every sliding
- * batch/window of the data stream at the given position.
- *
- * @param positionToMax
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
- dataStream.checkFieldRange(positionToMax);
- return aggregate(ComparableAggregator.getAggregator(positionToMax, getOutputType(),
- AggregationType.MAX));
- }
-
- /**
- * Applies an aggregation that gives the maximum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same maximum value the operator returns the first by default.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
- return this.maxBy(positionToMaxBy, true);
- }
-
- /**
- * Applies an aggregation that gives the maximum element of every sliding
- * batch/window of the data stream by the given position. If more elements
- * have the same maximum value the operator returns either the first or last
- * one depending on the parameter setting.
- *
- * @param positionToMaxBy
- * The position in the data point to maximize
- * @param first
- * If true, then the operator return the first element with the
- * maximum value, otherwise returns the last
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
- dataStream.checkFieldRange(positionToMaxBy);
- return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getOutputType(),
- AggregationType.MAXBY, first));
- }
-
- /**
- * Syntactic sugar for max(0)
- *
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max() {
- return max(0);
- }
-
- /**
- * Applies an aggregation that that gives the minimum of the pojo data
- * stream at the given field expression. A field expression is either the
- * name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> min(String field) {
- return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
- AggregationType.MIN, false));
- }
-
- /**
- * Applies an aggregation that that gives the maximum of the pojo data
- * stream at the given field expression. A field expression is either the
- * name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> max(String field) {
- return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
- AggregationType.MAX, false));
- }
-
- /**
- * Applies an aggregation that that gives the minimum element of the pojo
- * data stream by the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @param first
- * If True then in case of field equality the first object will
- * be returned
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
- return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
- AggregationType.MINBY, first));
- }
-
- /**
- * Applies an aggregation that that gives the maximum element of the pojo
- * data stream by the given field expression. A field expression is either
- * the name of a public field or a getter method with parentheses of the
- * {@link DataStream}S underlying type. A dot can be used to drill down into
- * objects, as in {@code "field1.getInnerField2()" }.
- *
- * @param field
- * The field expression based on which the aggregation will be
- * applied.
- * @param first
- * If True then in case of field equality the first object will
- * be returned
- * @return The transformed DataStream.
- */
- public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
- return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
- AggregationType.MAXBY, first));
- }
-
- /**
- * Gets the output type.
- *
- * @return The output type.
- */
- public TypeInformation<OUT> getOutputType() {
- return dataStream.getOutputType();
- }
-
- private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregate) {
- StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregate);
-
- SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.addFunction("batchReduce",
- aggregate, dataStream.outTypeWrapper, dataStream.outTypeWrapper, invokable);
-
- return returnStream;
- }
-
- protected BatchReduceInvokable<OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
- BatchReduceInvokable<OUT> invokable;
- if (isGrouped) {
- invokable = new GroupedBatchReduceInvokable<OUT>(reducer, batchSize, slideSize,
- keySelector);
- } else {
- invokable = new BatchReduceInvokable<OUT>(reducer, batchSize, slideSize);
- }
- return invokable;
- }
-
- protected <R> BatchGroupReduceInvokable<OUT, R> getGroupReduceInvokable(
- GroupReduceFunction<OUT, R> reducer) {
- BatchGroupReduceInvokable<OUT, R> invokable;
- if (isGrouped) {
- invokable = new GroupedBatchGroupReduceInvokable<OUT, R>(reducer, batchSize, slideSize,
- keySelector);
- } else {
- invokable = new BatchGroupReduceInvokable<OUT, R>(reducer, batchSize, slideSize);
- }
- return invokable;
- }
-
- protected BatchedDataStream<OUT> copy() {
- return new BatchedDataStream<OUT>(this);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index c38a638..2ce28d9 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -663,7 +663,6 @@ public class DataStream<OUT> {
return new GroupedDataStream<OUT>(this, keySelector);
}
-
/**
* This allows you to set up windowing through a nice API using
* {@link WindowingHelper} such as {@link Time}, {@link Count} and
@@ -678,101 +677,6 @@ public class DataStream<OUT> {
public WindowedDataStream<OUT> window(WindowingHelper... policyHelpers) {
return new WindowedDataStream<OUT>(this, policyHelpers);
}
-
- /**
- * Collects the data stream elements into sliding batches creating a new
- * {@link BatchedDataStream}. The user can apply transformations like
- * {@link BatchedDataStream#reduce}, {@link BatchedDataStream#reduceGroup}
- * or aggregations on the {@link BatchedDataStream}.
- *
- * @param batchSize
- * The number of elements in each batch at each operator
- * @param slideSize
- * The number of elements with which the batches are slid by
- * after each transformation.
- * @return The transformed {@link DataStream}
- */
- public BatchedDataStream<OUT> batch(long batchSize, long slideSize) {
- if (batchSize < 1) {
- throw new IllegalArgumentException("Batch size must be positive");
- }
- if (slideSize < 1) {
- throw new IllegalArgumentException("Slide size must be positive");
- }
- return new BatchedDataStream<OUT>(this, batchSize, slideSize);
- }
-
- /**
- * Collects the data stream elements into sliding batches creating a new
- * {@link BatchedDataStream}. The user can apply transformations like
- * {@link BatchedDataStream#reduce}, {@link BatchedDataStream#reduceGroup}
- * or aggregations on the {@link BatchedDataStream}.
- *
- * @param batchSize
- * The number of elements in each batch at each operator
- * @return The transformed {@link DataStream}
- */
- public BatchedDataStream<OUT> batch(long batchSize) {
- return batch(batchSize, batchSize);
- }
-
- /**
- * Collects the data stream elements into sliding windows creating a new
- * {@link WindowDataStream}. The user can apply transformations like
- * {@link WindowDataStream#reduce}, {@link WindowDataStream#reduceGroup} or
- * aggregations on the {@link WindowDataStream}.
- *
- * @param windowSize
- * The length of the window in milliseconds.
- * @param slideInterval
- * The number of milliseconds with which the windows are slid by
- * after each transformation.
- * @param timestamp
- * User defined function for extracting time-stamps from each
- * element
- * @return The transformed {@link DataStream}
- */
- public WindowDataStream<OUT> window(long windowSize, long slideInterval,
- TimeStamp<OUT> timestamp) {
- if (windowSize < 1) {
- throw new IllegalArgumentException("Window size must be positive");
- }
- if (slideInterval < 1) {
- throw new IllegalArgumentException("Slide interval must be positive");
- }
- return new WindowDataStream<OUT>(this, windowSize, slideInterval, timestamp);
- }
-
- /**
- * Collects the data stream elements into sliding windows creating a new
- * {@link WindowDataStream}. The user can apply transformations like
- * {@link WindowDataStream#reduce}, {@link WindowDataStream#reduceGroup} or
- * aggregations on the {@link WindowDataStream}.
- *
- * @param windowSize
- * The length of the window in milliseconds.
- * @param slideInterval
- * The number of milliseconds with which the windows are slid by
- * after each transformation.
- * @return The transformed {@link DataStream}
- */
- public WindowDataStream<OUT> window(long windowSize, long slideInterval) {
- return window(windowSize, slideInterval, new DefaultTimeStamp<OUT>());
- }
-
- /**
- * Collects the data stream elements into sliding windows creating a new
- * {@link WindowDataStream}. The user can apply transformations like
- * {@link WindowDataStream#reduce}, {@link WindowDataStream#reduceGroup} or
- * aggregations on the {@link WindowDataStream}.
- *
- * @param windowSize
- * The length of the window in milliseconds.
- * @return The transformed {@link DataStream}
- */
- public WindowDataStream<OUT> window(long windowSize) {
- return window(windowSize, windowSize);
- }
/**
* Applies an aggregation that sums the data stream at the given position.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDataStream.java
deleted file mode 100755
index be0e37f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowDataStream.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.datastream;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.BatchGroupReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.BatchReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowGroupReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowGroupReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowReduceInvokable;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-
-/**
- * A {@link WindowDataStream} represents a data stream whose elements are
- * batched together in a sliding window. operations like
- * {@link #reduce(ReduceFunction)} or {@link #reduceGroup(GroupReduceFunction)}
- * are applied for each window and the window is slid afterwards.
- *
- * @param <OUT>
- * The output type of the {@link WindowDataStream}
- */
-public class WindowDataStream<OUT> extends BatchedDataStream<OUT> {
-
- TimeStamp<OUT> timeStamp;
-
- protected WindowDataStream(DataStream<OUT> dataStream, long windowSize, long slideInterval,
- TimeStamp<OUT> timeStamp) {
- super(dataStream, windowSize, slideInterval);
- this.timeStamp = timeStamp;
- }
-
- protected WindowDataStream(WindowDataStream<OUT> windowDataStream) {
- super(windowDataStream);
- this.timeStamp = windowDataStream.timeStamp;
- }
-
- public WindowDataStream<OUT> groupBy(int... keyPositions) {
- return new WindowDataStream<OUT>(dataStream.groupBy(keyPositions), batchSize, slideSize,
- timeStamp);
- }
-
- public WindowDataStream<OUT> groupBy(String... fields) {
- return new WindowDataStream<OUT>(dataStream.groupBy(fields), batchSize, slideSize,
- timeStamp);
- }
-
- protected BatchReduceInvokable<OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
- WindowReduceInvokable<OUT> invokable;
- if (isGrouped) {
- invokable = new GroupedWindowReduceInvokable<OUT>(reducer, batchSize, slideSize,
- keySelector, timeStamp);
- } else {
- invokable = new WindowReduceInvokable<OUT>(reducer, batchSize, slideSize, timeStamp);
- }
- return invokable;
- }
-
- protected <R> BatchGroupReduceInvokable<OUT, R> getGroupReduceInvokable(
- GroupReduceFunction<OUT, R> reducer) {
- BatchGroupReduceInvokable<OUT, R> invokable;
- if (isGrouped) {
- invokable = new GroupedWindowGroupReduceInvokable<OUT, R>(reducer, batchSize,
- slideSize, keySelector, timeStamp);
- } else {
- invokable = new WindowGroupReduceInvokable<OUT, R>(reducer, batchSize, slideSize,
- timeStamp);
- }
- return invokable;
- }
-
- public WindowDataStream<OUT> copy() {
- return new WindowDataStream<OUT>(this);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 2e6b885..dd26e64 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -17,25 +17,34 @@
package org.apache.flink.streaming.api.datastream;
+import java.util.ArrayList;
import java.util.LinkedList;
+import java.util.List;
-import org.apache.commons.lang.ArrayUtils;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.invokable.operator.WindowingInvokable;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
+import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
+import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
+import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowingInvokable;
+import org.apache.flink.streaming.api.invokable.operator.WindowingGroupInvokable;
+import org.apache.flink.streaming.api.invokable.operator.WindowingReduceInvokable;
import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
-import org.apache.flink.streaming.util.serialization.CombineTypeWrapper;
import org.apache.flink.streaming.util.serialization.FunctionTypeWrapper;
-import org.apache.flink.streaming.util.serialization.ObjectTypeWrapper;
/**
- * A {@link WindowedDataStream} represents a data stream whose elements
- * are batched together in a sliding batch. operations like
+ * A {@link WindowedDataStream} represents a data stream whose elements are
+ * batched together in a sliding batch. operations like
* {@link #reduce(ReduceFunction)} or {@link #reduceGroup(GroupReduceFunction)}
* are applied for each batch and the batch is slid afterwards.
*
@@ -48,19 +57,31 @@ public class WindowedDataStream<OUT> {
protected boolean isGrouped;
protected KeySelector<OUT, ?> keySelector;
- protected WindowingHelper<OUT>[] triggerPolicies;
- protected WindowingHelper<OUT>[] evictionPolicies;
+ protected List<WindowingHelper<OUT>> triggerPolicies;
+ protected List<WindowingHelper<OUT>> evictionPolicies;
+
+ protected WindowedDataStream(DataStream<OUT> dataStream, WindowingHelper<OUT>... policyHelpers) {
+ this.dataStream = dataStream.copy();
+ this.triggerPolicies = new ArrayList<WindowingHelper<OUT>>();
+ for (WindowingHelper<OUT> helper : policyHelpers) {
+ this.triggerPolicies.add(helper);
+ }
- protected WindowedDataStream(DataStream<OUT> dataStream,
- WindowingHelper<OUT>... policyHelpers) {
if (dataStream instanceof GroupedDataStream) {
this.isGrouped = true;
this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+
} else {
this.isGrouped = false;
}
- this.dataStream = dataStream.copy();
- this.triggerPolicies = policyHelpers;
+ }
+
+ protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
+ this.dataStream = windowedDataStream.dataStream.copy();
+ this.isGrouped = windowedDataStream.isGrouped;
+ this.keySelector = windowedDataStream.keySelector;
+ this.triggerPolicies = windowedDataStream.triggerPolicies;
+ this.evictionPolicies = windowedDataStream.evictionPolicies;
}
protected LinkedList<TriggerPolicy<OUT>> getTriggers() {
@@ -87,18 +108,43 @@ public class WindowedDataStream<OUT> {
return evictionPolicyList;
}
- protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
- this.dataStream = windowedDataStream.dataStream.copy();
- this.isGrouped = windowedDataStream.isGrouped;
- this.keySelector = windowedDataStream.keySelector;
- this.triggerPolicies = windowedDataStream.triggerPolicies;
- this.evictionPolicies = windowedDataStream.evictionPolicies;
+ protected LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
+ LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
+ // Add Time triggers to central triggers
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (trigger instanceof TimeTriggerPolicy) {
+ cTriggers.add(trigger);
+ }
+ }
+ return cTriggers;
+ }
+
+ protected LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
+ LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
+
+ // Everything except Time triggers are distributed
+ for (TriggerPolicy<OUT> trigger : getTriggers()) {
+ if (!(trigger instanceof TimeTriggerPolicy)) {
+ dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
+ }
+ }
+
+ return dTriggers;
+ }
+
+ protected LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
+ LinkedList<CloneableEvictionPolicy<OUT>> evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
+ for (EvictionPolicy<OUT> evicter : getEvicters()) {
+ evicters.add((CloneableEvictionPolicy<OUT>) evicter);
+ }
+
+ return evicters;
}
/**
* Groups the elements of the {@link WindowedDataStream} by the given
- * key position to be used with grouped operators.
+ * {@link KeySelector} to be used with grouped operators.
*
* @param keySelector
* The specification of the key on which the
@@ -109,7 +155,39 @@ public class WindowedDataStream<OUT> {
WindowedDataStream<OUT> ret = this.copy();
ret.dataStream = ret.dataStream.groupBy(keySelector);
ret.isGrouped = true;
- ret.keySelector = keySelector;
+ ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
+ return ret;
+ }
+
+ /**
+ * Groups the elements of the {@link WindowedDataStream} by the given key
+ * positions to be used with grouped operators.
+ *
+ * @param fields
+ * The position of the fields to group by.
+ * @return The transformed {@link WindowedDataStream}
+ */
+ public WindowedDataStream<OUT> groupBy(int... fields) {
+ WindowedDataStream<OUT> ret = this.copy();
+ ret.dataStream = ret.dataStream.groupBy(fields);
+ ret.isGrouped = true;
+ ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
+ return ret;
+ }
+
+ /**
+ * Groups the elements of the {@link WindowedDataStream} by the given field
+ * expressions to be used with grouped operators.
+ *
+ * @param fields
+ * The position of the fields to group by.
+ * @return The transformed {@link WindowedDataStream}
+ */
+ public WindowedDataStream<OUT> groupBy(String... fields) {
+ WindowedDataStream<OUT> ret = this.copy();
+ ret.dataStream = ret.dataStream.groupBy(fields);
+ ret.isGrouped = true;
+ ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
return ret;
}
@@ -126,13 +204,242 @@ public class WindowedDataStream<OUT> {
* information
* @return The single output operator
*/
- public SingleOutputStreamOperator<Tuple2<OUT, String[]>, ?> reduce(
- ReduceFunction<OUT> reduceFunction) {
+ public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reduceFunction) {
return dataStream.addFunction("NextGenWindowReduce", reduceFunction,
- new FunctionTypeWrapper<OUT>(reduceFunction, ReduceFunction.class, 0),
- new CombineTypeWrapper<OUT, String[]>(dataStream.outTypeWrapper,
- new ObjectTypeWrapper<String[]>(new String[] { "" })),
- new WindowingInvokable<OUT>(reduceFunction, getTriggers(), getEvicters()));
+ dataStream.outTypeWrapper, dataStream.outTypeWrapper,
+ getReduceInvokable(reduceFunction));
+ }
+
+ public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
+ GroupReduceFunction<OUT, R> reduceFunction) {
+ return dataStream.addFunction("NextGenWindowReduce", reduceFunction,
+ dataStream.outTypeWrapper, new FunctionTypeWrapper<R>(reduceFunction,
+ GroupReduceFunction.class, 1), getReduceGroupInvokable(reduceFunction));
+ }
+
+ /**
+ * Applies an aggregation that sums every sliding batch/window of the data
+ * stream at the given position.
+ *
+ * @param positionToSum
+ * The position in the data point to sum
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
+ dataStream.checkFieldRange(positionToSum);
+ return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(positionToSum,
+ dataStream.getClassAtPos(positionToSum), dataStream.getOutputType()));
+ }
+
+ /**
+ * Syntactic sugar for sum(0)
+ *
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> sum() {
+ return sum(0);
+ }
+
+ /**
+ * Applies an aggregation that that gives the sum of the pojo data stream at
+ * the given field expression. A field expression is either the name of a
+ * public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> sum(String field) {
+ return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field,
+ getOutputType()));
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum of every sliding
+ * batch/window of the data stream at the given position.
+ *
+ * @param positionToMin
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
+ dataStream.checkFieldRange(positionToMin);
+ return aggregate(ComparableAggregator.getAggregator(positionToMin, getOutputType(),
+ AggregationType.MIN));
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum element of every sliding
+ * batch/window of the data stream by the given position. If more elements
+ * have the same minimum value the operator returns the first element by
+ * default.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
+ return this.minBy(positionToMinBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the minimum element of every sliding
+ * batch/window of the data stream by the given position. If more elements
+ * have the same minimum value the operator returns either the first or last
+ * one depending on the parameter setting.
+ *
+ * @param positionToMinBy
+ * The position in the data point to minimize
+ * @param first
+ * If true, then the operator return the first element with the
+ * minimum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
+ dataStream.checkFieldRange(positionToMinBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMinBy, getOutputType(),
+ AggregationType.MINBY, first));
+ }
+
+ /**
+ * Syntactic sugar for min(0)
+ *
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min() {
+ return min(0);
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum of every sliding
+ * batch/window of the data stream at the given position.
+ *
+ * @param positionToMax
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
+ dataStream.checkFieldRange(positionToMax);
+ return aggregate(ComparableAggregator.getAggregator(positionToMax, getOutputType(),
+ AggregationType.MAX));
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum element of every sliding
+ * batch/window of the data stream by the given position. If more elements
+ * have the same maximum value the operator returns the first by default.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
+ return this.maxBy(positionToMaxBy, true);
+ }
+
+ /**
+ * Applies an aggregation that gives the maximum element of every sliding
+ * batch/window of the data stream by the given position. If more elements
+ * have the same maximum value the operator returns either the first or last
+ * one depending on the parameter setting.
+ *
+ * @param positionToMaxBy
+ * The position in the data point to maximize
+ * @param first
+ * If true, then the operator return the first element with the
+ * maximum value, otherwise returns the last
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
+ dataStream.checkFieldRange(positionToMaxBy);
+ return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getOutputType(),
+ AggregationType.MAXBY, first));
+ }
+
+ /**
+ * Syntactic sugar for max(0)
+ *
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max() {
+ return max(0);
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum of the pojo data
+ * stream at the given field expression. A field expression is either the
+ * name of a public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> min(String field) {
+ return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
+ AggregationType.MIN, false));
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum of the pojo data
+ * stream at the given field expression. A field expression is either the
+ * name of a public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> max(String field) {
+ return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
+ AggregationType.MAX, false));
+ }
+
+ /**
+ * Applies an aggregation that that gives the minimum element of the pojo
+ * data stream by the given field expression. A field expression is either
+ * the name of a public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @param first
+ * If True then in case of field equality the first object will
+ * be returned
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
+ return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
+ AggregationType.MINBY, first));
+ }
+
+ /**
+ * Applies an aggregation that that gives the maximum element of the pojo
+ * data stream by the given field expression. A field expression is either
+ * the name of a public field or a getter method with parentheses of the
+ * {@link DataStream}S underlying type. A dot can be used to drill down into
+ * objects, as in {@code "field1.getInnerField2()" }.
+ *
+ * @param field
+ * The field expression based on which the aggregation will be
+ * applied.
+ * @param first
+ * If True then in case of field equality the first object will
+ * be returned
+ * @return The transformed DataStream.
+ */
+ public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
+ return aggregate(ComparableAggregator.getAggregator(field, getOutputType(),
+ AggregationType.MAXBY, first));
}
/**
@@ -153,11 +460,45 @@ public class WindowedDataStream<OUT> {
WindowedDataStream<OUT> ret = this.copy();
if (ret.evictionPolicies == null) {
ret.evictionPolicies = ret.triggerPolicies;
- ret.triggerPolicies = policyHelpers;
- } else {
- ret.triggerPolicies = (WindowingHelper<OUT>[]) ArrayUtils.addAll(triggerPolicies,
- policyHelpers);
+ ret.triggerPolicies = new ArrayList<WindowingHelper<OUT>>();
+ }
+ for (WindowingHelper<OUT> helper : policyHelpers) {
+ ret.triggerPolicies.add(helper);
}
return ret;
}
+
+ private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregator) {
+ StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregator);
+
+ SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.addFunction("windowReduce",
+ aggregator, dataStream.outTypeWrapper, dataStream.outTypeWrapper, invokable);
+
+ return returnStream;
+ }
+
+ protected <R> StreamInvokable<OUT, R> getReduceGroupInvokable(
+ GroupReduceFunction<OUT, R> reducer) {
+ StreamInvokable<OUT, R> invokable;
+ if (isGrouped) {
+ invokable = new GroupedWindowingInvokable<OUT, R>(reducer, keySelector,
+ getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers());
+
+ } else {
+ invokable = new WindowingGroupInvokable<OUT, R>(reducer, getTriggers(), getEvicters());
+ }
+ return invokable;
+ }
+
+ protected StreamInvokable<OUT, OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
+ StreamInvokable<OUT, OUT> invokable;
+ if (isGrouped) {
+ invokable = new GroupedWindowingInvokable<OUT, OUT>(reducer, keySelector,
+ getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers());
+
+ } else {
+ invokable = new WindowingReduceInvokable<OUT>(reducer, getTriggers(), getEvicters());
+ }
+ return invokable;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceInvokable.java
deleted file mode 100755
index 134c33d..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceInvokable.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.io.Serializable;
-
-import org.apache.commons.math.util.MathUtils;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.state.CircularFifoList;
-
-public class BatchGroupReduceInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
- protected GroupReduceFunction<IN, OUT> reducer;
-
- protected long slideSize;
-
- protected long batchSize;
- protected int granularity;
- protected int batchPerSlide;
- protected StreamBatch batch;
- protected StreamBatch currentBatch;
- protected long numberOfBatches;
-
- public BatchGroupReduceInvokable(GroupReduceFunction<IN, OUT> reduceFunction, long batchSize,
- long slideSize) {
- super(reduceFunction);
- this.reducer = reduceFunction;
- this.batchSize = batchSize;
- this.slideSize = slideSize;
- this.granularity = (int) MathUtils.gcd(batchSize, slideSize);
- this.batchPerSlide = (int) (slideSize / granularity);
- this.numberOfBatches = batchSize / granularity;
- this.batch = new StreamBatch();
- }
-
- @Override
- protected void immutableInvoke() throws Exception {
- if ((reuse = recordIterator.next(reuse)) == null) {
- throw new RuntimeException("DataStream must not be empty");
- }
-
- while (reuse != null) {
- StreamBatch batch = getBatch(reuse);
- batch.addToBuffer(reuse.getObject());
-
- resetReuse();
- reuse = recordIterator.next(reuse);
- }
-
- reduceLastBatch();
- }
-
- @Override
- // TODO: implement mutableInvoke for reduce
- protected void mutableInvoke() throws Exception {
- System.out.println("Immutable setting is used");
- immutableInvoke();
- }
-
- protected StreamBatch getBatch(StreamRecord<IN> next) {
- return batch;
- }
-
- protected void reduce(StreamBatch batch) {
- this.currentBatch = batch;
- callUserFunctionAndLogException();
- }
-
- protected void reduceLastBatch() {
- batch.reduceLastBatch();
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- if(!currentBatch.circularList.isEmpty()){
- reducer.reduce(currentBatch.circularList.getIterable(), collector);
- }
- }
-
- protected class StreamBatch implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private long counter;
- protected long minibatchCounter;
-
- protected CircularFifoList<IN> circularList;
-
- public StreamBatch() {
- this.circularList = new CircularFifoList<IN>();
- this.counter = 0;
- this.minibatchCounter = 0;
- }
-
- public void addToBuffer(IN nextValue) throws Exception {
- circularList.add(nextValue);
-
- counter++;
-
- if (miniBatchEnd()) {
- circularList.newSlide();
- minibatchCounter++;
- if (batchEnd()) {
- reduceBatch();
- circularList.shiftWindow(batchPerSlide);
- }
- }
-
- }
-
- protected boolean miniBatchEnd() {
- if( (counter % granularity) == 0){
- counter = 0;
- return true;
- }else{
- return false;
- }
- }
-
-
- public boolean batchEnd() {
- if (minibatchCounter == numberOfBatches) {
- minibatchCounter -= batchPerSlide;
- return true;
- }
- return false;
- }
-
- public void reduceBatch() {
- reduce(this);
- }
-
- public void reduceLastBatch() {
- if (!miniBatchEnd()) {
- reduceBatch();
- }
- }
-
- @Override
- public String toString(){
- return circularList.toString();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceInvokable.java
deleted file mode 100755
index 6b22555..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceInvokable.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-import org.apache.commons.math.util.MathUtils;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.state.NullableCircularBuffer;
-
-public class BatchReduceInvokable<OUT> extends StreamInvokable<OUT, OUT> {
-
- private static final long serialVersionUID = 1L;
- protected ReduceFunction<OUT> reducer;
-
- protected long slideSize;
-
- protected long batchSize;
- protected int granularity;
- protected long batchPerSlide;
- protected long numberOfBatches;
- protected StreamBatch batch;
- protected StreamBatch currentBatch;
-
- protected TypeSerializer<OUT> serializer;
-
- public BatchReduceInvokable(ReduceFunction<OUT> reduceFunction, long batchSize, long slideSize) {
- super(reduceFunction);
- this.reducer = reduceFunction;
- this.batchSize = batchSize;
- this.slideSize = slideSize;
- this.granularity = (int) MathUtils.gcd(batchSize, slideSize);
- this.batchPerSlide = slideSize / granularity;
- this.numberOfBatches = batchSize / granularity;
- }
-
- @Override
- protected void immutableInvoke() throws Exception {
- if ((reuse = recordIterator.next(reuse)) == null) {
- throw new RuntimeException("DataStream must not be empty");
- }
-
- while (reuse != null) {
- StreamBatch batch = getBatch(reuse);
-
- batch.reduceToBuffer(reuse.getObject());
-
- resetReuse();
- reuse = recordIterator.next(reuse);
- }
-
- reduceLastBatch();
-
- }
-
- @Override
- protected void mutableInvoke() throws Exception {
- System.out.println("Immutable setting is used");
- immutableInvoke();
- }
-
- protected StreamBatch getBatch(StreamRecord<OUT> next) {
- return batch;
- }
-
- protected void reduce(StreamBatch batch) {
- this.currentBatch = batch;
- callUserFunctionAndLogException();
- }
-
- protected void reduceLastBatch() throws Exception {
- batch.reduceLastBatch();
- }
-
- @Override
- protected void callUserFunction() throws Exception {
- Iterator<OUT> reducedIterator = currentBatch.getIterator();
- OUT reduced = null;
-
- while (reducedIterator.hasNext() && reduced == null) {
- reduced = reducedIterator.next();
- }
-
- while (reducedIterator.hasNext()) {
- OUT next = reducedIterator.next();
- if (next != null) {
- reduced = reducer.reduce(serializer.copy(reduced), serializer.copy(next));
- }
- }
- if (reduced != null) {
- collector.collect(reduced);
- }
- }
-
- protected class StreamBatch implements Serializable {
-
- private static final long serialVersionUID = 1L;
- protected long counter;
- protected long minibatchCounter;
- protected OUT currentValue;
- boolean changed;
-
- protected NullableCircularBuffer circularBuffer;
-
- public StreamBatch() {
-
- this.circularBuffer = new NullableCircularBuffer((int) (batchSize / granularity));
- this.counter = 0;
- this.minibatchCounter = 0;
- this.changed = false;
- }
-
- public void reduceToBuffer(OUT nextValue) throws Exception {
-
- if (currentValue != null) {
- currentValue = reducer.reduce(serializer.copy(currentValue), serializer.copy(nextValue));
- } else {
- currentValue = nextValue;
- }
-
- counter++;
-
- if (miniBatchEnd()) {
- addToBuffer();
- if (batchEnd()) {
- reduceBatch();
- }
- }
-
- }
-
- protected void addToBuffer() {
- circularBuffer.add(currentValue);
- changed = true;
- minibatchCounter++;
- currentValue = null;
- }
-
- protected boolean miniBatchEnd() {
- if ((counter % granularity) == 0) {
- counter = 0;
- return true;
- } else {
- return false;
- }
- }
-
- public boolean batchEnd() {
- if (minibatchCounter == numberOfBatches) {
- minibatchCounter -= batchPerSlide;
- return true;
- }
- return false;
- }
-
- public void reduceLastBatch() throws Exception {
-
- if (miniBatchInProgress()) {
- addToBuffer();
- }
- if (changed == true && minibatchCounter >= 0) {
- if (circularBuffer.isFull()) {
- for (long i = 0; i < (numberOfBatches - minibatchCounter); i++) {
- if (!circularBuffer.isEmpty()) {
- circularBuffer.remove();
- }
- }
- }
- if (!circularBuffer.isEmpty()) {
- reduce(this);
- }
- }
- }
-
- public boolean miniBatchInProgress() {
- return currentValue != null;
- }
-
- public void reduceBatch() {
- reduce(this);
- changed = false;
- }
-
- @SuppressWarnings("unchecked")
- public Iterator<OUT> getIterator() {
- return circularBuffer.iterator();
- }
-
- @Override
- public String toString() {
- return circularBuffer.toString();
- }
-
- }
-
-
- @Override
- public void open(Configuration config) throws Exception{
- super.open(config);
- serializer = inSerializer.getObjectSerializer();
- this.batch = new StreamBatch();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceInvokable.java
deleted file mode 100755
index 3564829..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceInvokable.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-public class GroupedBatchGroupReduceInvokable<IN, OUT> extends BatchGroupReduceInvokable<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- Map<Object, StreamBatch> streamBatches;
- KeySelector<IN, ?> keySelector;
-
-
- public GroupedBatchGroupReduceInvokable(GroupReduceFunction<IN, OUT> reduceFunction, long batchSize,
- long slideSize, KeySelector<IN, ?> keySelector) {
- super(reduceFunction, batchSize, slideSize);
- this.keySelector = keySelector;
- this.streamBatches = new HashMap<Object, StreamBatch>();
- }
-
- @Override
- protected StreamBatch getBatch(StreamRecord<IN> next) {
- Object key = next.getKey(keySelector);
- StreamBatch batch = streamBatches.get(key);
- if(batch == null){
- batch=new StreamBatch();
- streamBatches.put(key, batch);
- }
- return batch;
- }
-
- @Override
- protected void reduceLastBatch() {
- for(StreamBatch batch: streamBatches.values()){
- batch.reduceLastBatch();
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceInvokable.java
deleted file mode 100755
index cfb128c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceInvokable.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-public class GroupedBatchReduceInvokable<OUT> extends BatchReduceInvokable<OUT> {
-
- private static final long serialVersionUID = 1L;
- KeySelector<OUT, ?> keySelector;
- Map<Object, StreamBatch> streamBatches;
-
- public GroupedBatchReduceInvokable(ReduceFunction<OUT> reduceFunction, long batchSize,
- long slideSize, KeySelector<OUT, ?> keySelector) {
- super(reduceFunction, batchSize, slideSize);
- this.keySelector = keySelector;
- this.streamBatches = new HashMap<Object, StreamBatch>();
- }
-
- @Override
- protected StreamBatch getBatch(StreamRecord<OUT> next) {
- Object key = next.getKey(keySelector);
- StreamBatch batch = streamBatches.get(key);
- if (batch == null) {
- batch = new StreamBatch();
- streamBatches.put(key, batch);
- }
- return batch;
- }
-
- @Override
- protected void reduceLastBatch() throws Exception {
- for (StreamBatch batch : streamBatches.values()) {
- batch.reduceLastBatch();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokable.java
deleted file mode 100755
index 0641ae3..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokable.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-public class GroupedWindowGroupReduceInvokable<IN, OUT> extends WindowGroupReduceInvokable<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
-
- KeySelector<IN, ?> keySelector;
- Map<Object, StreamWindow> streamWindows;
- List<Object> cleanList;
- long currentMiniBatchCount = 0;
-
- public GroupedWindowGroupReduceInvokable(GroupReduceFunction<IN, OUT> reduceFunction,
- long windowSize, long slideInterval, KeySelector<IN, ?> keySelector,
- TimeStamp<IN> timestamp) {
- super(reduceFunction, windowSize, slideInterval, timestamp);
- this.keySelector = keySelector;
- this.reducer = reduceFunction;
- this.streamWindows = new HashMap<Object, StreamWindow>();
- }
-
- @Override
- protected StreamBatch getBatch(StreamRecord<IN> next) {
- Object key = next.getKey(keySelector);
- StreamWindow window = streamWindows.get(key);
- if (window == null) {
- window = new GroupedStreamWindow();
- for (int i = 0; i < currentMiniBatchCount; i++) {
- window.circularList.newSlide();
- }
- streamWindows.put(key, window);
- }
- this.window = window;
- return window;
- }
-
- @Override
- protected void reduceLastBatch() {
- for (StreamBatch window : streamWindows.values()) {
- window.reduceLastBatch();
- }
- }
-
- private void shiftGranularityAllWindows() {
- for (StreamBatch window : streamWindows.values()) {
- window.circularList.newSlide();
- }
- }
-
- private void slideAllWindows() {
- currentMiniBatchCount -= batchPerSlide;
- for (StreamBatch window : streamWindows.values()) {
- window.circularList.shiftWindow(batchPerSlide);
- }
- }
-
- private void reduceAllWindows() {
- for (StreamBatch window : streamWindows.values()) {
- window.reduceBatch();
- }
- }
-
- protected class GroupedStreamWindow extends StreamWindow {
-
- private static final long serialVersionUID = 1L;
-
- public GroupedStreamWindow() {
- super();
- }
-
- @Override
- public void addToBuffer(IN nextValue) throws Exception {
- checkWindowEnd(timestamp.getTimestamp(nextValue));
- if (currentMiniBatchCount >= 0) {
- circularList.add(nextValue);
- }
- }
-
- @Override
- protected synchronized void checkWindowEnd(long timeStamp) {
- nextRecordTime = timeStamp;
-
- while (miniBatchEnd()) {
- shiftGranularityAllWindows();
- currentMiniBatchCount += 1;
- if (batchEnd()) {
- reduceAllWindows();
- slideAllWindows();
- }
- }
- }
-
- @Override
- public boolean batchEnd() {
- if (currentMiniBatchCount == numberOfBatches) {
- return true;
- }
- return false;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowReduceInvokable.java
deleted file mode 100644
index 2248096..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowReduceInvokable.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-public class GroupedWindowReduceInvokable<OUT> extends WindowReduceInvokable<OUT> {
-
- private static final long serialVersionUID = 1L;
- private KeySelector<OUT, ?> keySelector;
- private Map<Object, StreamWindow> streamWindows;
- private long currentMiniBatchCount = 0;
-
- public GroupedWindowReduceInvokable(ReduceFunction<OUT> reduceFunction, long windowSize,
- long slideInterval, KeySelector<OUT, ?> keySelector, TimeStamp<OUT> timestamp) {
- super(reduceFunction, windowSize, slideInterval, timestamp);
- this.keySelector = keySelector;
- this.streamWindows = new HashMap<Object, StreamWindow>();
- }
-
- @Override
- protected StreamBatch getBatch(StreamRecord<OUT> next) {
- Object key = next.getKey(keySelector);
- StreamWindow window = streamWindows.get(key);
- if (window == null) {
- window = new GroupedStreamWindow();
- window.minibatchCounter = currentMiniBatchCount;
- streamWindows.put(key, window);
- }
- this.window = window;
- return window;
- }
-
- private void addToAllBuffers() {
- for (StreamBatch window : streamWindows.values()) {
- window.addToBuffer();
- }
- }
-
- private void reduceAllWindows() {
- for (StreamBatch window : streamWindows.values()) {
- window.minibatchCounter -= batchPerSlide;
- window.reduceBatch();
- }
- }
-
- @Override
- protected void reduceLastBatch() throws Exception {
- for (StreamBatch window : streamWindows.values()) {
- window.reduceLastBatch();
- }
- }
-
- protected class GroupedStreamWindow extends StreamWindow {
-
- private static final long serialVersionUID = 1L;
-
- public GroupedStreamWindow() {
- super();
- }
-
- @Override
- protected synchronized void checkWindowEnd(long timeStamp) {
- nextRecordTime = timeStamp;
-
- while (miniBatchEnd()) {
- addToAllBuffers();
- if (batchEnd()) {
- reduceAllWindows();
- }
- }
- currentMiniBatchCount = this.minibatchCounter;
- }
-
- @Override
- public boolean batchEnd() {
- if (minibatchCounter == numberOfBatches) {
- return true;
- }
- return false;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
index 7fc35ca..e957b8c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
@@ -21,9 +21,10 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
@@ -69,7 +70,7 @@ import org.slf4j.LoggerFactory;
* @param <IN>
* The type of input elements handled by this operator invokable.
*/
-public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, String[]>> {
+public class GroupedWindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
/**
* Auto-generated serial version UID
@@ -84,7 +85,7 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
- private Map<Object, WindowingInvokable<IN>> windowingGroups = new HashMap<Object, WindowingInvokable<IN>>();
+ private Map<Object, WindowingInvokable<IN, OUT>> windowingGroups = new HashMap<Object, WindowingInvokable<IN, OUT>>();
private LinkedList<Thread> activePolicyThreads = new LinkedList<Thread>();
private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
@@ -117,7 +118,7 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
* that it forwards/distributed calls all groups.
*
* @param userFunction
- * The user defined {@link ReduceFunction}.
+ * The user defined function.
* @param keySelector
* A key selector to extract the key for the groups from the
* input data.
@@ -137,7 +138,7 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
* If only one element is contained a group, this element itself
* is returned as aggregated result.)
*/
- public GroupedWindowingInvokable(ReduceFunction<IN> userFunction,
+ public GroupedWindowingInvokable(Function userFunction,
KeySelector<IN, ?> keySelector,
LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
@@ -165,8 +166,8 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
// Continuously run
while (reuse != null) {
- WindowingInvokable<IN> groupInvokable = windowingGroups.get(keySelector.getKey(reuse
- .getObject()));
+ WindowingInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector
+ .getKey(reuse.getObject()));
if (groupInvokable == null) {
groupInvokable = makeNewGroup(reuse);
}
@@ -175,7 +176,7 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
IN[] result = trigger.preNotifyTrigger(reuse.getObject());
for (IN in : result) {
- for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(in, trigger);
}
}
@@ -193,7 +194,7 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
groupInvokable.processRealElement(reuse.getObject());
} else {
// call user function for all groups
- for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
if (group == groupInvokable) {
// process real with initialized policies
group.processRealElement(reuse.getObject(), currentTriggerPolicies);
@@ -219,7 +220,7 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
}
// finally trigger the buffer.
- for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
group.emitFinalWindow(centralTriggerPolicies);
}
@@ -239,7 +240,8 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
* {@link KeySelector#getKey(Object)}, the exception is not
* catched by this method.
*/
- private WindowingInvokable<IN> makeNewGroup(StreamRecord<IN> element) throws Exception {
+ @SuppressWarnings("unchecked")
+ private WindowingInvokable<IN, OUT> makeNewGroup(StreamRecord<IN> element) throws Exception {
// clone the policies
LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
@@ -250,10 +252,17 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
clonedDistributedEvictionPolicies.add(eviction.clone());
}
- @SuppressWarnings("unchecked")
- WindowingInvokable<IN> groupInvokable = new WindowingInvokable<IN>(
- (ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
- clonedDistributedEvictionPolicies);
+ WindowingInvokable<IN, OUT> groupInvokable;
+ if (userFunction instanceof ReduceFunction) {
+ groupInvokable = (WindowingInvokable<IN, OUT>) new WindowingReduceInvokable<IN>(
+ (ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
+ clonedDistributedEvictionPolicies);
+ } else {
+ groupInvokable = new WindowingGroupInvokable<IN, OUT>(
+ (GroupReduceFunction<IN, OUT>) userFunction, clonedDistributedTriggerPolicies,
+ clonedDistributedEvictionPolicies);
+ }
+
groupInvokable.initialize(collector, recordIterator, inSerializer, isMutable);
groupInvokable.open(this.parameters);
windowingGroups.put(keySelector.getKey(element.getObject()), groupInvokable);
@@ -305,7 +314,7 @@ public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN
@Override
public void sendFakeElement(IN datapoint) {
- for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ for (WindowingInvokable<IN, OUT> group : windowingGroups.values()) {
group.processFakeElement(datapoint, policy);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
deleted file mode 100755
index c69816d4..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-
-public class WindowGroupReduceInvokable<IN, OUT> extends BatchGroupReduceInvokable<IN, OUT> {
-
- private static final long serialVersionUID = 1L;
- private long startTime;
- protected long nextRecordTime;
- protected TimeStamp<IN> timestamp;
- protected StreamWindow window;
-
- public WindowGroupReduceInvokable(GroupReduceFunction<IN, OUT> reduceFunction, long windowSize,
- long slideInterval, TimeStamp<IN> timestamp) {
- super(reduceFunction, windowSize, slideInterval);
- this.timestamp = timestamp;
- this.startTime = timestamp.getStartTime();
- this.window = new StreamWindow();
- this.batch = this.window;
- }
-
- @Override
- public void open(Configuration config) throws Exception {
- super.open(config);
- if (timestamp instanceof DefaultTimeStamp) {
- (new TimeCheck()).start();
- }
- }
-
- protected class StreamWindow extends StreamBatch {
-
- private static final long serialVersionUID = 1L;
-
- public StreamWindow() {
- super();
- }
-
- @Override
- public void addToBuffer(IN nextValue) throws Exception {
- checkWindowEnd(timestamp.getTimestamp(nextValue));
- if (minibatchCounter >= 0) {
- circularList.add(nextValue);
- }
- }
-
- protected synchronized void checkWindowEnd(long timeStamp) {
- nextRecordTime = timeStamp;
-
- while (miniBatchEnd()) {
- circularList.newSlide();
- minibatchCounter++;
- if (batchEnd()) {
- reduceBatch();
- circularList.shiftWindow(batchPerSlide);
- }
- }
- }
-
- @Override
- protected boolean miniBatchEnd() {
- if (nextRecordTime < startTime + granularity) {
- return false;
- } else {
- startTime += granularity;
- return true;
- }
- }
-
- }
-
- private class TimeCheck extends Thread {
- @Override
- public void run() {
- while (true) {
- try {
- Thread.sleep(slideSize);
- } catch (InterruptedException e) {
- }
- if (isRunning) {
- window.checkWindowEnd(System.currentTimeMillis());
- } else {
- break;
- }
- }
- }
- }
-
-}
[24/34] incubator-flink git commit: [streaming] Changed time based
windowing policies to make lower boundary excluded and upper boundary
included in the windows and adjusted policy test cases respectively.
Posted by mb...@apache.org.
[streaming] Changed time based windowing policies to make lower boundary excluded and upper boundary included in the windows and adjusted policy test cases respectively.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/996f02cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/996f02cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/996f02cb
Branch: refs/heads/master
Commit: 996f02cbb7a89fde3803a664c9977d7f83f50a40
Parents: 0ef399c
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Tue Nov 25 12:48:37 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../streaming/api/windowing/policy/TimeEvictionPolicy.java | 2 +-
.../streaming/api/windowing/policy/TimeTriggerPolicy.java | 6 +++---
.../streaming/api/windowing/policy/TimeEvictionPolicyTest.java | 2 +-
.../streaming/api/windowing/policy/TimeTriggerPolicyTest.java | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/996f02cb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
index e886c09..99116d0 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
@@ -114,7 +114,7 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
int counter = 0;
while (!buffer.isEmpty()) {
- if (timestamp.getTimestamp(buffer.getFirst()) < threshold) {
+ if (timestamp.getTimestamp(buffer.getFirst()) <= threshold) {
buffer.removeFirst();
counter++;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/996f02cb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
index c3402c9..3539ad6 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -109,7 +109,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
// use > here. In case >= would fit, the regular call will do the job.
while (timestamp.getTimestamp(datapoint) > startTime + granularity) {
startTime += granularity;
- fakeElements.add(startTime);
+ fakeElements.add(startTime-1);
}
return (Object[]) fakeElements.toArray();
}
@@ -143,9 +143,10 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
*/
private synchronized void activeFakeElementEmission(ActiveTriggerCallback callback) {
+ // start time is excluded, but end time is included: >=
if (System.currentTimeMillis() >= startTime + granularity) {
startTime += granularity;
- callback.sendFakeElement(startTime);
+ callback.sendFakeElement(startTime-1);
}
}
@@ -177,7 +178,6 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
@Override
public synchronized boolean notifyTrigger(DATA datapoint) {
long recordTime = timestamp.getTimestamp(datapoint);
- // start time is included, but end time is excluded: >=
if (recordTime >= startTime + granularity) {
if (granularity != 0) {
startTime = recordTime - ((recordTime - startTime) % granularity);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/996f02cb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
index 303c66a..82c8841 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicyTest.java
@@ -89,7 +89,7 @@ public class TimeEvictionPolicyTest {
// handle correctness of eviction
for (; result > 0 && !buffer.isEmpty(); result--) {
- if (buffer.getFirst() < times[i] - granularity) {
+ if (buffer.getFirst() <= times[i] - granularity) {
buffer.removeFirst();
} else {
fail("The policy wanted to evict time " + buffer.getFirst()
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/996f02cb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
index 1355b27..28d35ec 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
@@ -105,7 +105,7 @@ public class TimeTriggerPolicyTest {
timeStamp);
// expected result
- Long[][] result = { {}, {}, { 5L, 10L, 15L }, { 25L } };
+ Long[][] result = { {}, {}, { 4L, 9L, 14L }, { 24L } };
// call policy
for (int i = 0; i < times.length; i++) {
[28/34] incubator-flink git commit: [FLINK-1279] [streaming] Forward
partitioning changed to use round-robin method
Posted by mb...@apache.org.
[FLINK-1279] [streaming] Forward partitioning changed to use round-robin method
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/f0cc5d64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/f0cc5d64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/f0cc5d64
Branch: refs/heads/master
Commit: f0cc5d64b4361845aebef43124b77da51e0b7824
Parents: 64baa00
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Nov 27 13:17:41 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:10 2014 +0100
----------------------------------------------------------------------
.../flink/streaming/api/JobGraphBuilder.java | 7 ++--
.../streaming/api/datastream/DataStream.java | 14 ++-----
.../api/datastream/IterativeDataStream.java | 6 +--
.../partitioner/BroadcastPartitioner.java | 10 ++++-
.../partitioner/DistributePartitioner.java | 20 ++++-----
.../partitioner/FieldsPartitioner.java | 6 +--
.../partitioner/ForwardPartitioner.java | 43 --------------------
.../partitioner/GlobalPartitioner.java | 8 ++--
.../partitioner/ShufflePartitioner.java | 7 ++--
.../partitioner/StreamPartitioner.java | 27 ++++++++----
.../apache/flink/streaming/api/PrintTest.java | 1 +
.../partitioner/DistributePartitionerTest.java | 2 +-
.../partitioner/ForwardPartitionerTest.java | 16 ++++----
13 files changed, 65 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
index 03198ce..8a8595a 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/JobGraphBuilder.java
@@ -37,8 +37,8 @@ import org.apache.flink.streaming.api.streamvertex.CoStreamVertex;
import org.apache.flink.streaming.api.streamvertex.StreamIterationHead;
import org.apache.flink.streaming.api.streamvertex.StreamIterationTail;
import org.apache.flink.streaming.api.streamvertex.StreamVertex;
-import org.apache.flink.streaming.partitioner.ForwardPartitioner;
import org.apache.flink.streaming.partitioner.StreamPartitioner;
+import org.apache.flink.streaming.partitioner.StreamPartitioner.PartitioningStrategy;
import org.apache.flink.streaming.state.OperatorState;
import org.apache.flink.streaming.util.serialization.TypeWrapper;
import org.slf4j.Logger;
@@ -80,7 +80,6 @@ public class JobGraphBuilder {
private Map<String, Long> iterationWaitTime;
private Map<String, Map<String, OperatorState<?>>> operatorStates;
-
/**
* Creates an new {@link JobGraph} with the given name. A JobGraph is a DAG
* and consists of sources, tasks (intermediate vertices) and sinks.
@@ -205,7 +204,7 @@ public class JobGraphBuilder {
*/
public void addIterationTail(String vertexName, String iterationTail, String iterationID,
int parallelism, long waitTime) {
-
+
if (bufferTimeout.get(iterationTail) == 0) {
throw new RuntimeException("Buffer timeout 0 at iteration tail is not supported.");
}
@@ -355,7 +354,7 @@ public class JobGraphBuilder {
StreamConfig config = new StreamConfig(upStreamVertex.getConfiguration());
- if (partitionerObject.getClass().equals(ForwardPartitioner.class)) {
+ if (partitionerObject.getStrategy() == PartitioningStrategy.FORWARD) {
downStreamVertex
.connectNewDataSetAsInput(upStreamVertex, DistributionPattern.POINTWISE);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index fd19f73..55f5e71 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -66,7 +66,6 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.partitioner.BroadcastPartitioner;
import org.apache.flink.streaming.partitioner.DistributePartitioner;
import org.apache.flink.streaming.partitioner.FieldsPartitioner;
-import org.apache.flink.streaming.partitioner.ForwardPartitioner;
import org.apache.flink.streaming.partitioner.ShufflePartitioner;
import org.apache.flink.streaming.partitioner.StreamPartitioner;
import org.apache.flink.streaming.util.keys.FieldsKeySelector;
@@ -127,7 +126,7 @@ public class DataStream<OUT> {
this.jobGraphBuilder = environment.getJobGraphBuilder();
this.userDefinedNames = new ArrayList<String>();
this.selectAll = false;
- this.partitioner = new ForwardPartitioner<OUT>();
+ this.partitioner = new DistributePartitioner<OUT>(true);
this.outTypeWrapper = outTypeWrapper;
this.mergedStreams = new ArrayList<DataStream<OUT>>();
this.mergedStreams.add(this);
@@ -159,13 +158,6 @@ public class DataStream<OUT> {
}
/**
- * Partitioning strategy on the stream.
- */
- public static enum ConnectionType {
- SHUFFLE, BROADCAST, FIELD, FORWARD, DISTRIBUTE
- }
-
- /**
* Returns the ID of the {@link DataStream}.
*
* @return ID of the DataStream
@@ -341,7 +333,7 @@ public class DataStream<OUT> {
* @return The DataStream with shuffle partitioning set.
*/
public DataStream<OUT> forward() {
- return setConnectionType(new ForwardPartitioner<OUT>());
+ return setConnectionType(new DistributePartitioner<OUT>(true));
}
/**
@@ -351,7 +343,7 @@ public class DataStream<OUT> {
* @return The DataStream with shuffle partitioning set.
*/
public DataStream<OUT> distribute() {
- return setConnectionType(new DistributePartitioner<OUT>());
+ return setConnectionType(new DistributePartitioner<OUT>(false));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index 41616c9..29f5eec 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -20,7 +20,7 @@ package org.apache.flink.streaming.api.datastream;
import java.util.Arrays;
import java.util.List;
-import org.apache.flink.streaming.partitioner.ForwardPartitioner;
+import org.apache.flink.streaming.partitioner.DistributePartitioner;
/**
* The iterative data stream represents the start of an iteration in a
@@ -91,8 +91,8 @@ public class IterativeDataStream<IN> extends
for (DataStream<IN> stream : iterationTail.mergedStreams) {
String inputID = stream.getId();
- jobGraphBuilder.setEdge(inputID, returnStream.getId(), new ForwardPartitioner<IN>(), 0,
- name, false);
+ jobGraphBuilder.setEdge(inputID, returnStream.getId(), new DistributePartitioner<IN>(
+ true), 0, name, false);
}
return iterationTail;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
index d8abea6..3813c8a 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/BroadcastPartitioner.java
@@ -26,16 +26,21 @@ import org.apache.flink.streaming.api.streamrecord.StreamRecord;
* @param <T>
* Type of the Tuple
*/
-public class BroadcastPartitioner<T> implements StreamPartitioner<T> {
+public class BroadcastPartitioner<T> extends StreamPartitioner<T> {
private static final long serialVersionUID = 1L;
int[] returnArray;
boolean set;
+ int setNumber;
+
+ public BroadcastPartitioner() {
+ super(PartitioningStrategy.BROADCAST);
+ }
@Override
public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
int numberOfOutputChannels) {
- if (set) {
+ if (set && setNumber == numberOfOutputChannels) {
return returnArray;
} else {
this.returnArray = new int[numberOfOutputChannels];
@@ -43,6 +48,7 @@ public class BroadcastPartitioner<T> implements StreamPartitioner<T> {
returnArray[i] = i;
}
set = true;
+ setNumber = numberOfOutputChannels;
return returnArray;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
index 6e95c5f..7299397 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/DistributePartitioner.java
@@ -27,22 +27,20 @@ import org.apache.flink.streaming.api.streamrecord.StreamRecord;
* @param <T>
* Type of the Tuple
*/
-public class DistributePartitioner<T> implements StreamPartitioner<T> {
+public class DistributePartitioner<T> extends StreamPartitioner<T> {
private static final long serialVersionUID = 1L;
- private int currentChannelIndex;
- private int[] returnArray;
-
- public DistributePartitioner() {
- this.currentChannelIndex = 0;
- this.returnArray = new int[1];
+ private int[] returnArray = new int[] {-1};
+
+ public DistributePartitioner(boolean forward) {
+ super(forward ? PartitioningStrategy.FORWARD : PartitioningStrategy.DISTRIBUTE);
}
-
+
@Override
public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
int numberOfOutputChannels) {
- returnArray[0] = currentChannelIndex;
- currentChannelIndex = (currentChannelIndex + 1) % numberOfOutputChannels;
- return returnArray;
+ this.returnArray[0] = (this.returnArray[0] + 1) % numberOfOutputChannels;
+
+ return this.returnArray;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
index 0b6a8cc..e61555d 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/FieldsPartitioner.java
@@ -28,15 +28,15 @@ import org.apache.flink.streaming.api.streamrecord.StreamRecord;
* @param <T>
* Type of the Tuple
*/
-public class FieldsPartitioner<T> implements StreamPartitioner<T> {
+public class FieldsPartitioner<T> extends StreamPartitioner<T> {
private static final long serialVersionUID = 1L;
- private int[] returnArray;
+ private int[] returnArray = new int[1];;
KeySelector<T, ?> keySelector;
public FieldsPartitioner(KeySelector<T, ?> keySelector) {
+ super(PartitioningStrategy.FIELDS);
this.keySelector = keySelector;
- this.returnArray = new int[1];
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ForwardPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ForwardPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ForwardPartitioner.java
deleted file mode 100755
index d9ca822..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ForwardPartitioner.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.partitioner;
-
-import org.apache.flink.runtime.plugable.SerializationDelegate;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-
-/**
- * Partitioner that forwards the tuples to the local subtask of the output vertex
- *
- * @param <T>
- * Type of the Tuple
- */
-public class ForwardPartitioner<T> implements StreamPartitioner<T> {
- private static final long serialVersionUID = 1L;
-
- private int[] returnArray;
-
- public ForwardPartitioner() {
- this.returnArray = new int[]{0};
- }
-
- @Override
- public int[] selectChannels(SerializationDelegate<StreamRecord<T>> record,
- int numberOfOutputChannels) {
- return returnArray;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
index 73b171e..c73ca71 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/GlobalPartitioner.java
@@ -21,13 +21,13 @@ import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
//Group to the partitioner with the lowest id
-public class GlobalPartitioner<T> implements StreamPartitioner<T> {
+public class GlobalPartitioner<T> extends StreamPartitioner<T> {
private static final long serialVersionUID = 1L;
-
- private int[] returnArray;
+
+ private int[] returnArray = new int[] { 0 };
public GlobalPartitioner() {
- this.returnArray = new int[] { 0 };
+ super(PartitioningStrategy.GLOBAL);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
index e5bfeb7..318de3f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/ShufflePartitioner.java
@@ -29,16 +29,15 @@ import org.apache.flink.streaming.api.streamrecord.StreamRecord;
* @param <T>
* Type of the Tuple
*/
-public class ShufflePartitioner<T> implements StreamPartitioner<T> {
+public class ShufflePartitioner<T> extends StreamPartitioner<T> {
private static final long serialVersionUID = 1L;
private Random random = new Random();
- private int[] returnArray;
+ private int[] returnArray = new int[1];
public ShufflePartitioner() {
- this.random = new Random();
- this.returnArray = new int[1];
+ super(PartitioningStrategy.SHUFFLE);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
index fb920f0..15b4f43 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/partitioner/StreamPartitioner.java
@@ -22,12 +22,23 @@ import org.apache.flink.runtime.io.network.api.ChannelSelector;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-/**
- * Empty interface to encapsulate partitioners.
- *
- * @param <T>
- * Type of the Tuple
- */
-public interface StreamPartitioner<T> extends ChannelSelector<SerializationDelegate<StreamRecord<T>>>,
- Serializable {
+public abstract class StreamPartitioner<T> implements
+ ChannelSelector<SerializationDelegate<StreamRecord<T>>>, Serializable {
+
+ public enum PartitioningStrategy {
+
+ FORWARD, DISTRIBUTE, SHUFFLE, BROADCAST, GLOBAL, FIELDS;
+
+ }
+
+ private static final long serialVersionUID = 1L;
+ private PartitioningStrategy strategy;
+
+ public StreamPartitioner(PartitioningStrategy strategy) {
+ this.strategy = strategy;
+ }
+
+ public PartitioningStrategy getStrategy() {
+ return strategy;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
index dd8652e..d1d5b1e 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
@@ -27,6 +27,7 @@ import org.junit.Test;
public class PrintTest implements Serializable {
+ private static final long serialVersionUID = 1L;
private static final long MEMORYSIZE = 32;
private static final class IdentityMap implements MapFunction<Long, Long> {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
index 6c0a3f6..0675242 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/DistributePartitionerTest.java
@@ -34,7 +34,7 @@ public class DistributePartitionerTest {
@Before
public void setPartitioner() {
- distributePartitioner = new DistributePartitioner<Tuple>();
+ distributePartitioner = new DistributePartitioner<Tuple>(false);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f0cc5d64/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
index 440cfcd..b381d85 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/partitioner/ForwardPartitionerTest.java
@@ -26,17 +26,17 @@ import org.junit.Before;
import org.junit.Test;
public class ForwardPartitionerTest {
-
- private ForwardPartitioner<Tuple> forwardPartitioner;
+
+ private DistributePartitioner<Tuple> forwardPartitioner;
private StreamRecord<Tuple> streamRecord = new StreamRecord<Tuple>();
private SerializationDelegate<StreamRecord<Tuple>> sd = new SerializationDelegate<StreamRecord<Tuple>>(
null);
-
+
@Before
public void setPartitioner() {
- forwardPartitioner = new ForwardPartitioner<Tuple>();
+ forwardPartitioner = new DistributePartitioner<Tuple>(true);
}
-
+
@Test
public void testSelectChannelsLength() {
sd.setInstance(streamRecord);
@@ -44,12 +44,12 @@ public class ForwardPartitionerTest {
assertEquals(1, forwardPartitioner.selectChannels(sd, 2).length);
assertEquals(1, forwardPartitioner.selectChannels(sd, 1024).length);
}
-
+
@Test
public void testSelectChannelsInterval() {
sd.setInstance(streamRecord);
assertEquals(0, forwardPartitioner.selectChannels(sd, 1)[0]);
- assertEquals(0, forwardPartitioner.selectChannels(sd, 2)[0]);
- assertEquals(0, forwardPartitioner.selectChannels(sd, 1024)[0]);
+ assertEquals(1, forwardPartitioner.selectChannels(sd, 2)[0]);
+ assertEquals(2, forwardPartitioner.selectChannels(sd, 1024)[0]);
}
}
[06/34] incubator-flink git commit: [streaming] Created trigger and
eviction policies based on element counters. Included respective policy
helper and test cases.
Posted by mb...@apache.org.
[streaming] Created trigger and eviction policies based on element counters. Included respective policy helper and test cases.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/d731b3b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/d731b3b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/d731b3b0
Branch: refs/heads/master
Commit: d731b3b06bfb7d61b0464715f22482c34ad88ff6
Parents: 93178ae
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:22:26 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../streaming/api/windowing/helper/Count.java | 68 +++++++++++
.../windowing/policy/CountEvictionPolicy.java | 115 ++++++++++++++++++
.../windowing/policy/CountTriggerPolicy.java | 83 +++++++++++++
.../policy/CountEvictionPolicyTest.java | 119 +++++++++++++++++++
.../policy/CountTriggerPolicyTest.java | 94 +++++++++++++++
5 files changed, 479 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d731b3b0/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
new file mode 100644
index 0000000..b74e952
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.helper;
+
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * Represents a count based trigger or eviction policy.
+ * Use the {@link Count#of(int)} to get an instance.
+ */
+@SuppressWarnings("rawtypes")
+public class Count implements WindowingHelper {
+
+ private int count;
+
+ /**
+ * Specifies on which element a trigger or an eviction should happen (based
+ * on the count of the elements).
+ *
+ * This constructor does exactly the same as {@link Count#of(int)}.
+ *
+ * @param count the number of elements to count before trigger/evict
+ */
+ public Count(int count) {
+ this.count = count;
+ }
+
+ @Override
+ public EvictionPolicy<?> toEvict() {
+ return new CountEvictionPolicy(count);
+ }
+
+ @Override
+ public TriggerPolicy<?> toTrigger() {
+ return new CountTriggerPolicy(count);
+ }
+
+ /**
+ * Specifies on which element a trigger or an eviction should happen (based
+ * on the count of the elements)
+ *
+ * @param count
+ * the number of elements to count before trigger/evict
+ * @return An helper representing the policy
+ */
+ public static Count of(int count) {
+ return new Count(count);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d731b3b0/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
new file mode 100644
index 0000000..1d8175c
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicy.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This eviction policy allows the eviction of data points from the buffer using
+ * a counter of arriving elements and a threshold (maximal buffer size)
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it count even fake elements.
+ *
+ * @param <IN>
+ * the type of the incoming data points
+ */
+public class CountEvictionPolicy<IN> implements EvictionPolicy<IN> {
+
+ /**
+ * Auto generated version id
+ */
+ private static final long serialVersionUID = 2319201348806427996L;
+
+ int maxElements;
+ int counter;
+ int deleteOnEviction = 1;
+
+ /**
+ * This constructor allows the setup of the simplest possible count based
+ * eviction. It keeps the size of the buffer according to the given
+ * maxElements parameter by deleting the oldest element in the buffer.
+ * Eviction only takes place if the counter of arriving elements would be
+ * higher than maxElements without eviction.
+ *
+ * @param maxElements
+ * The maximum number of elements before eviction. As soon as one
+ * more element arrives, the oldest element will be deleted
+ */
+ public CountEvictionPolicy(int maxElements) {
+ this(maxElements, 1);
+ }
+
+ /**
+ * This constructor allows to set up both, the maximum number of elements
+ * and the number of elements to be deleted in case of an eviction.
+ *
+ * Eviction only takes place if the counter of arriving elements would be
+ * higher than maxElements without eviction. In such a case deleteOnEviction
+ * elements will be removed from the buffer.
+ *
+ * The counter of arriving elements is adjusted respectively, but never set
+ * below zero:
+ * counter=(counter-deleteOnEviction<0)?0:counter-deleteOnEviction
+ *
+ * @param maxElements
+ * maxElements The maximum number of elements before eviction.
+ * @param deleteOnEviction
+ * The number of elements to be deleted on eviction. The counter
+ * will be adjusted respectively but never below zero.
+ */
+ public CountEvictionPolicy(int maxElements, int deleteOnEviction) {
+ this(maxElements, deleteOnEviction, 0);
+ }
+
+ /**
+ * The same as {@link CountEvictionPolicy#CountEvictionPolicy(int, int)}.
+ * Additionally a custom start value for the counter of arriving elements
+ * can be set. By setting a negative start value the first eviction can be
+ * delayed.
+ *
+ * @param maxElements
+ * maxElements The maximum number of elements before eviction.
+ * @param deleteOnEviction
+ * The number of elements to be deleted on eviction. The counter
+ * will be adjusted respectively but never below zero.
+ * @param startValue
+ * A custom start value for the counter of arriving elements.
+ * @see CountEvictionPolicy#NextGenCountEvictionPolicy(int, int)
+ */
+ public CountEvictionPolicy(int maxElements, int deleteOnEviction, int startValue) {
+ this.counter = startValue;
+ this.deleteOnEviction = deleteOnEviction;
+ this.maxElements = maxElements;
+ }
+
+ @Override
+ public int notifyEviction(IN datapoint, boolean triggered, int bufferSize) {
+ // The comparison have to be >= and not == to cover case max=0
+ if (counter >= maxElements) {
+ // Adjust the counter according to the current eviction
+ counter = (counter - deleteOnEviction < 0) ? 0 : counter - deleteOnEviction;
+ // The current element will be added after the eviction
+ // Therefore, increase counter in any case
+ counter++;
+ return deleteOnEviction;
+ } else {
+ counter++;
+ return 0;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d731b3b0/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
new file mode 100644
index 0000000..ca0058e
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+/**
+ * This policy triggers at every n'th element.
+ *
+ * @param <IN>
+ * The type of the data points which are handled by this policy
+ */
+public class CountTriggerPolicy<IN> implements TriggerPolicy<IN> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -6357200688886103968L;
+
+ private static final int DEFAULT_START_VALUE = 0;
+
+ private int counter;
+ private int max;
+
+ /**
+ * This constructor will set up a count based trigger, which triggers after
+ * max elements have arrived.
+ *
+ * @param max
+ * The number of arriving elements before the trigger occurs.
+ */
+ public CountTriggerPolicy(int max) {
+ this(max, DEFAULT_START_VALUE);
+ }
+
+ /**
+ * In addition to {@link CountTriggerPolicy#CountTriggerPolicy(int)} this
+ * constructor allows to set a custom start value for the element counter.
+ * This can be used to delay the first trigger by setting a negative start
+ * value. Often the first trigger should be delayed in case of sliding
+ * windows. For example if the size of a window should be 4 and a trigger
+ * should happen every 2, a start value of -2 would allow to also have the
+ * first window of size 4.
+ *
+ * @param max
+ * The number of arriving elements before the trigger occurs.
+ * @param startValue
+ * The start value for the counter of arriving elements.
+ * @see CountTriggerPolicy#CountTriggerPolicy(int)
+ */
+ public CountTriggerPolicy(int max, int startValue) {
+ this.max = max;
+ this.counter = startValue;
+ }
+
+ @Override
+ public boolean notifyTrigger(IN datapoint) {
+ // The comparison have to be >= and not == to cover case max=0
+ if (counter >= max) {
+ // The current data point will be part of the next window!
+ // Therefore the counter needs to be set to one already.
+ counter = 1;
+ return true;
+ } else {
+ counter++;
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d731b3b0/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
new file mode 100644
index 0000000..4773f4e
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountEvictionPolicyTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class CountEvictionPolicyTest {
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ @Test
+ public void testCountEvictionPolicy() {
+ List<Integer> tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ int counter;
+
+ // The count policy should not care about the triggered parameter
+ // Therefore its value switches after each use in this test.
+ boolean triggered = false;
+ // the size of the buffer should not matter as well!
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+ EvictionPolicy evictionPolicy = new CountEvictionPolicy(i, i);
+ counter = 0;
+
+ // Test first i steps (should not evict)
+ for (int j = 0; j < i; j++) {
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i + " evicted tuples at add nr. "
+ + counter + ". It should not evict for the first " + i + " adds.", 0,
+ evictionPolicy.notifyEviction(tuples.get(j), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+
+ // Test the next three evictions
+ for (int j = 0; j < 3; j++) {
+ // The first add should evict now
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " did not evict correct number of tuples at the expected pos " + counter
+ + ".", i, evictionPolicy.notifyEviction(tuples.get(j),
+ (triggered = !triggered), tuples.get(Math.abs((i - j)) % 10)));
+
+ // the next i-1 adds should not evict
+ for (int k = 0; k < i - 1; k++) {
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " evicted tuples at add nr. " + counter, 0,
+ evictionPolicy.notifyEviction(tuples.get(j), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCountEvictionPolicyStartValuesAndEvictionAmount() {
+
+ // The count policy should not care about the triggered parameter
+ // Therefore its value switches after each use in this test.
+ boolean triggered = false;
+ // the size of the buffer should not matter as well!
+
+ List<Integer> tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ // Text different eviction amounts (0..3)
+ for (int x = 0; x < 4; x++) {
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+
+ int counter = 0;
+
+ // Test different start values (-5..5)
+ for (int j = -5; i < 6; i++) {
+ EvictionPolicy evictionPolicy = new CountEvictionPolicy(i, x, j);
+ // Add tuples without eviction
+ for (int k = 0; k < ((i - j > 0) ? i - j : 0); k++) {
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " did not evict correct number of tuples at the expected pos "
+ + counter + ".", 0, evictionPolicy.notifyEviction(
+ tuples.get(Math.abs(j)), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+ // Expect eviction
+ counter++;
+ assertEquals("Evictionpolicy with count of " + i
+ + " did not evict correct number of tuples at the expected pos "
+ + counter + ".", x, evictionPolicy.notifyEviction(
+ tuples.get(Math.abs(j)), (triggered = !triggered),
+ tuples.get(Math.abs((i - j)) % 10)));
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/d731b3b0/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
new file mode 100644
index 0000000..b7120e7
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicyTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.junit.Test;
+
+public class CountTriggerPolicyTest {
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCountTriggerPolicy() {
+
+ List tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+ int counter;
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+ TriggerPolicy triggerPolicy = Count.of(i).toTrigger();
+ counter=0;
+
+ // Test first i steps (should not trigger)
+ for (int j = 0; j < i; j++) {
+ counter++;
+ assertFalse("Triggerpolicy with count of " + i + " triggered at add nr. " + counter
+ + ". It should not trigger for the first " + i + " adds.",
+ triggerPolicy.notifyTrigger(tuples.get(j)));
+ }
+
+ // Test the next three triggers
+ for (int j = 0; j < 3; j++) {
+ // The first add should trigger now
+ counter++;
+ assertTrue("Triggerpolicy with count of " + i
+ + " did not trigger at the expected pos " + counter + ".",
+ triggerPolicy.notifyTrigger(tuples.get(j)));
+
+ // the next i-1 adds should not trigger
+ for (int k = 0; k < i - 1; k++) {
+ counter++;
+ assertFalse("Triggerpolicy with count of " + i + " triggered at add nr. "
+ + counter, triggerPolicy.notifyTrigger(tuples.get(k)));
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testCountTriggerPolicyStartValues() {
+
+ List tuples = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+ // Test count of different sizes (0..9)
+ for (int i = 0; i < 10; i++) {
+
+ // Test different start values (-5..5)
+ for (int j = -5; i < 6; i++) {
+ TriggerPolicy triggerPolicy = new CountTriggerPolicy(i, j);
+ // Add tuples without trigger
+ for (int k = 0; k < ((i - j > 0) ? i - j : 0); k++) {
+ assertFalse("Triggerpolicy with count of " + i + " and start value of " + j
+ + " triggered at add nr. " + (k + 1),
+ triggerPolicy.notifyTrigger(tuples.get(k % 10)));
+ }
+ // Expect trigger
+ assertTrue("Triggerpolicy with count of " + i + "and start value of " + j
+ + " did not trigger at the expected position.",
+ triggerPolicy.notifyTrigger(tuples.get(0)));
+ }
+ }
+ }
+}
[13/34] incubator-flink git commit: [streaming] New windowing API
merge and cleanup + several minor fixes
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
index 069693f..9b242f6 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/MultiplePoliciesExample.java
@@ -17,35 +17,27 @@
package org.apache.flink.streaming.examples.windowing;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.helper.Count;
import org.apache.flink.util.Collector;
-import java.util.LinkedList;
-
/**
* This example uses count based tumbling windowing with multiple eviction
* policies at the same time.
*/
public class MultiplePoliciesExample {
- private static final int PARALLELISM = 1;
+ private static final int PARALLELISM = 2;
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment
.createLocalEnvironment(PARALLELISM);
- LinkedList<TriggerPolicy<String>> policies = new LinkedList<TriggerPolicy<String>>();
- policies.add(new CountTriggerPolicy<String>(5));
- policies.add(new CountTriggerPolicy<String>(8));
-
// This reduce function does a String concat.
- ReduceFunction<String> reducer = new ReduceFunction<String>() {
+ GroupReduceFunction<String, String> reducer = new GroupReduceFunction<String, String>() {
/**
* Auto generates version ID
@@ -53,14 +45,21 @@ public class MultiplePoliciesExample {
private static final long serialVersionUID = 1L;
@Override
- public String reduce(String value1, String value2) throws Exception {
- return value1 + "|" + value2;
+ public void reduce(Iterable<String> values, Collector<String> out) throws Exception {
+ String output = "|";
+ for (String v : values) {
+ output = output + v + "|";
+ }
+ out.collect(output);
}
};
- DataStream<Tuple2<String, String[]>> stream = env.addSource(new BasicSource()).window(
- policies, reducer);
+ DataStream<String> stream = env.addSource(new BasicSource())
+ .groupBy(0)
+ .window(Count.of(2))
+ .every(Count.of(3), Count.of(5))
+ .reduceGroup(reducer);
stream.print();
@@ -70,13 +69,16 @@ public class MultiplePoliciesExample {
public static class BasicSource implements SourceFunction<String> {
private static final long serialVersionUID = 1L;
- String str = new String("streaming");
+
+ String str1 = new String("streaming");
+ String str2 = new String("flink");
@Override
public void invoke(Collector<String> out) throws Exception {
// continuous emit
while (true) {
- out.collect(str);
+ out.collect(str1);
+ out.collect(str2);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
index 1768480..c9c78b5 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/SlidingExample.java
@@ -18,14 +18,10 @@
package org.apache.flink.streaming.examples.windowing;
import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.helper.Count;
import org.apache.flink.util.Collector;
/**
@@ -45,19 +41,18 @@ public class SlidingExample {
* SIMPLE-EXAMPLE: Use this to always keep the newest 10 elements in the
* buffer Resulting windows will have an overlap of 5 elements
*/
- // TriggerPolicy<String> triggerPolicy=new
- // CountTriggerPolicy<String>(5);
- // EvictionPolicy<String> evictionPolicy=new
- // CountEvictionPolicy<String>(10);
-
+
+ // DataStream<String> stream = env.addSource(new CountingSource())
+ // .window(Count.of(10))
+ // .every(Count.of(5))
+ // .reduce(reduceFunction);
+
/*
* ADVANCED-EXAMPLE: Use this to have the last element of the last
* window as first element of the next window while the window size is
* always 5
*/
- TriggerPolicy<String> triggerPolicy = new CountTriggerPolicy<String>(4, -1);
- EvictionPolicy<String> evictionPolicy = new CountEvictionPolicy<String>(5, 4);
-
+
// This reduce function does a String concat.
ReduceFunction<String> reduceFunction = new ReduceFunction<String>() {
@@ -73,8 +68,10 @@ public class SlidingExample {
};
- DataStream<Tuple2<String, String[]>> stream = env.addSource(new CountingSource()).window(
- triggerPolicy, evictionPolicy, reduceFunction);
+ DataStream<String> stream = env.addSource(new CountingSource())
+ .window(Count.of(5).withDelete(4))
+ .every(Count.of(4).startingAt(-1))
+ .reduce(reduceFunction);
stream.print();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
index 115363b..fba73be 100644
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TimeWindowingExample.java
@@ -17,27 +17,26 @@
package org.apache.flink.streaming.examples.windowing;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.api.windowing.helper.Time;
import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.util.Collector;
/**
* This example shows the functionality of time based windows. It utilizes the
- * {@link ActiveTriggerPolicy} implementation in the {@link TimeTriggerPolicy}.
+ * {@link ActiveTriggerPolicy} implementation in the
+ * {@link ActiveTimeTriggerPolicy}.
*/
public class TimeWindowingExample {
- private static final int PARALLELISM = 1;
+ private static final int PARALLELISM = 2;
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment
@@ -46,38 +45,21 @@ public class TimeWindowingExample {
// Prevent output from being blocked
env.setBufferTimeout(100);
- // Trigger every 1000ms
- TriggerPolicy<Integer> triggerPolicy = new TimeTriggerPolicy<Integer>(1000L,
- new DefaultTimeStamp<Integer>(), new Extractor<Long, Integer>() {
+ DataStream<Integer> stream = env.addSource(new CountingSourceWithSleep())
+ .groupBy(new KeySelector<Integer, Integer>() {
private static final long serialVersionUID = 1L;
@Override
- public Integer extract(Long in) {
- return in.intValue();
+ public Integer getKey(Integer value) throws Exception {
+ if (value < 3) {
+ return 0;
+ } else {
+ return 1;
+ }
}
- });
-
- // Always keep the newest 100 elements in the buffer
- EvictionPolicy<Integer> evictionPolicy = new CountEvictionPolicy<Integer>(100);
-
- // This reduce function does a String concat.
- ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
-
- /**
- * default version ID
- */
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
-
- };
-
- DataStream<Tuple2<Integer, String[]>> stream = env.addSource(new CountingSourceWithSleep()).window(triggerPolicy, evictionPolicy, reduceFunction);
+ }).window(Count.of(100)).every(Time.of(1000, TimeUnit.MILLISECONDS)).sum();
stream.print();
@@ -97,6 +79,7 @@ public class TimeWindowingExample {
@Override
public void invoke(Collector<Integer> collector) throws Exception {
+ Random rnd = new Random();
// continuous emit
while (true) {
if (counter > 9999) {
@@ -105,7 +88,7 @@ public class TimeWindowingExample {
System.out.println("Source continouse with emitting now!");
counter = 0;
}
- collector.collect(counter);
+ collector.collect(rnd.nextInt(9) + 1);
// Wait 0.001 sec. before the next emit. Otherwise the source is
// too fast for local tests and you might always see
[03/34] incubator-flink git commit: [streaming] Introduced extractor
interface to enable flexible handling of arbitrary input data types.
Posted by mb...@apache.org.
[streaming] Introduced extractor interface to enable flexible handling of arbitrary input data types.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/40c3ae48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/40c3ae48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/40c3ae48
Branch: refs/heads/master
Commit: 40c3ae481f538cebe7ecb265a796474e07517eee
Parents: d731b3b
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:29:38 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../api/windowing/extractor/Extractor.java | 43 ++++++++++++++++++++
1 file changed, 43 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/40c3ae48/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
new file mode 100644
index 0000000..b103ca3
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/extractor/Extractor.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.extractor;
+
+import java.io.Serializable;
+
+/**
+ * Extractors allow to extract/convert one type to another. They are mostly used
+ * to extract some fields out of a more complex structure (Tuple/Array) to run
+ * further calculation on the extraction result.
+ *
+ * @param <FROM>
+ * The input data type.
+ * @param <TO>
+ * The output data type.
+ */
+public interface Extractor<FROM, TO> extends Serializable {
+
+ /**
+ * Extracts/Converts the given input to an object of the output type
+ *
+ * @param in
+ * the input data
+ * @return the extracted/converted data
+ */
+ public TO extract(FROM in);
+
+}
[14/34] incubator-flink git commit: [streaming] New windowing API
merge and cleanup + several minor fixes
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
deleted file mode 100755
index 0135b34..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-
-public class WindowReduceInvokable<OUT> extends BatchReduceInvokable<OUT> {
- private static final long serialVersionUID = 1L;
- protected long startTime;
- protected long nextRecordTime;
- protected TimeStamp<OUT> timestamp;
- protected StreamWindow window;
-
- public WindowReduceInvokable(ReduceFunction<OUT> reduceFunction, long windowSize,
- long slideInterval, TimeStamp<OUT> timestamp) {
- super(reduceFunction, windowSize, slideInterval);
- this.timestamp = timestamp;
- this.startTime = timestamp.getStartTime();
- }
-
- @Override
- public void open(Configuration config) throws Exception {
- super.open(config);
- this.window = new StreamWindow();
- this.batch = this.window;
- if (timestamp instanceof DefaultTimeStamp) {
- (new TimeCheck()).start();
- }
- }
-
- protected class StreamWindow extends StreamBatch {
-
- private static final long serialVersionUID = 1L;
-
- public StreamWindow() {
- super();
-
- }
-
- @Override
- public void reduceToBuffer(OUT nextValue) throws Exception {
-
- checkWindowEnd(timestamp.getTimestamp(nextValue));
-
- if (currentValue != null) {
- currentValue = reducer.reduce(serializer.copy(currentValue), serializer.copy(nextValue));
- } else {
- currentValue = nextValue;
- }
- }
-
- protected synchronized void checkWindowEnd(long timeStamp) {
- nextRecordTime = timeStamp;
-
- while (miniBatchEnd()) {
- addToBuffer();
- if (batchEnd()) {
- reduceBatch();
- }
- }
- }
-
- @Override
- public void reduceBatch() {
- reduce(this);
- }
-
- @Override
- protected boolean miniBatchEnd() {
- if (nextRecordTime < startTime + granularity) {
- return false;
- } else {
- startTime += granularity;
- return true;
- }
- }
-
- @Override
- public boolean batchEnd() {
- if (minibatchCounter == numberOfBatches) {
- minibatchCounter -= batchPerSlide;
- return true;
- }
- return false;
- }
-
- }
-
- private class TimeCheck extends Thread {
- @Override
- public void run() {
- while (true) {
- try {
- Thread.sleep(slideSize);
- } catch (InterruptedException e) {
- }
- if (isRunning) {
- window.checkWindowEnd(System.currentTimeMillis());
- } else {
- break;
- }
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java
new file mode 100644
index 0000000..2752fd1
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingGroupInvokable.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.LinkedList;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public class WindowingGroupInvokable<IN, OUT> extends WindowingInvokable<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+ GroupReduceFunction<IN, OUT> reducer;
+
+ public WindowingGroupInvokable(GroupReduceFunction<IN, OUT> userFunction,
+ LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction, triggerPolicies, evictionPolicies);
+ this.reducer = userFunction;
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ reducer.reduce(buffer, collector);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
index 29975f4..d0855e6 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
@@ -17,8 +17,12 @@
package org.apache.flink.streaming.api.invokable.operator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.invokable.StreamInvokable;
import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
@@ -28,12 +32,7 @@ import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, String[]>> {
+public abstract class WindowingInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
/**
* Auto-generated serial version UID
@@ -47,9 +46,8 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
private LinkedList<Thread> activePolicyTreads = new LinkedList<Thread>();
- private LinkedList<IN> buffer = new LinkedList<IN>();
+ protected LinkedList<IN> buffer = new LinkedList<IN>();
private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
- private ReduceFunction<IN> reducer;
/**
* This constructor created a windowing invokable using trigger and eviction
@@ -64,12 +62,10 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
* A list of {@link EvictionPolicy}s and/or
* {@link ActiveEvictionPolicy}s
*/
- public WindowingInvokable(ReduceFunction<IN> userFunction,
- LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ public WindowingInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
LinkedList<EvictionPolicy<IN>> evictionPolicies) {
super(userFunction);
- this.reducer = userFunction;
this.triggerPolicies = triggerPolicies;
this.evictionPolicies = evictionPolicies;
@@ -369,28 +365,4 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
}
}
- @Override
- protected void callUserFunction() throws Exception {
- Iterator<IN> reducedIterator = buffer.iterator();
- IN reduced = null;
-
- while (reducedIterator.hasNext() && reduced == null) {
- reduced = reducedIterator.next();
- }
-
- while (reducedIterator.hasNext()) {
- IN next = reducedIterator.next();
- if (next != null) {
- reduced = reducer.reduce(reduced, next);
- }
- }
- if (reduced != null) {
- String[] tmp = new String[currentTriggerPolicies.size()];
- for (int i = 0; i < tmp.length; i++) {
- tmp[i] = currentTriggerPolicies.get(i).toString();
- }
- collector.collect(new Tuple2<IN, String[]>(reduced, tmp));
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java
new file mode 100644
index 0000000..89f98e0
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingReduceInvokable.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public class WindowingReduceInvokable<IN> extends WindowingInvokable<IN, IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ ReduceFunction<IN> reducer;
+
+ public WindowingReduceInvokable(ReduceFunction<IN> userFunction,
+ LinkedList<TriggerPolicy<IN>> triggerPolicies,
+ LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+ super(userFunction, triggerPolicies, evictionPolicies);
+ this.reducer = userFunction;
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ Iterator<IN> reducedIterator = buffer.iterator();
+ IN reduced = null;
+
+ while (reducedIterator.hasNext() && reduced == null) {
+ reduced = reducedIterator.next();
+ }
+
+ while (reducedIterator.hasNext()) {
+ IN next = reducedIterator.next();
+ if (next != null) {
+ reduced = reducer.reduce(reduced, next);
+ }
+ }
+ if (reduced != null) {
+ collector.collect(reduced);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
index b74e952..6e045f6 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Count.java
@@ -23,21 +23,24 @@ import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
/**
- * Represents a count based trigger or eviction policy.
- * Use the {@link Count#of(int)} to get an instance.
+ * Represents a count based trigger or eviction policy. Use the
+ * {@link Count#of(int)} to get an instance.
*/
@SuppressWarnings("rawtypes")
public class Count implements WindowingHelper {
private int count;
+ private int deleteOnEviction = 1;
+ private int startValue = CountTriggerPolicy.DEFAULT_START_VALUE;
/**
* Specifies on which element a trigger or an eviction should happen (based
* on the count of the elements).
*
- * This constructor does exactly the same as {@link Count#of(int)}.
+ * This constructor does exactly the same as {@link Count#of(int)}.
*
- * @param count the number of elements to count before trigger/evict
+ * @param count
+ * the number of elements to count before trigger/evict
*/
public Count(int count) {
this.count = count;
@@ -45,12 +48,22 @@ public class Count implements WindowingHelper {
@Override
public EvictionPolicy<?> toEvict() {
- return new CountEvictionPolicy(count);
+ return new CountEvictionPolicy(count, deleteOnEviction);
}
@Override
public TriggerPolicy<?> toTrigger() {
- return new CountTriggerPolicy(count);
+ return new CountTriggerPolicy(count, startValue);
+ }
+
+ public Count withDelete(int deleteOnEviction) {
+ this.deleteOnEviction = deleteOnEviction;
+ return this;
+ }
+
+ public Count startingAt(int startValue) {
+ this.startValue = startValue;
+ return this;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
index 37f2902..0b73150 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/helper/Time.java
@@ -20,10 +20,11 @@ package org.apache.flink.streaming.api.windowing.helper;
import java.util.concurrent.TimeUnit;
import org.apache.flink.streaming.api.invokable.util.DefaultTimeStamp;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
/**
@@ -39,7 +40,9 @@ public class Time<DATA> implements WindowingHelper<DATA> {
private int timeVal;
private TimeUnit granularity;
- private Extractor<Long, DATA> timeToData;
+ private Extractor<Long, DATA> longToDATAExtractor;
+ private TimeStamp<DATA> timeStamp;
+ private long delay;
/**
* Creates an helper representing a trigger which triggers every given
@@ -47,50 +50,29 @@ public class Time<DATA> implements WindowingHelper<DATA> {
*
* @param timeVal
* The number of time units
- * @param granularity
+ * @param timeUnit
* The unit of time such as minute oder millisecond. Note that
* the smallest possible granularity is milliseconds. Any smaller
* time unit might cause an error at runtime due to conversion
* problems.
- * @param timeToData
- * This policy creates fake elements to not miss windows in case
- * no element arrived within the duration of the window. This
- * extractor should wrap a long into such an element of type
- * DATA.
*/
- public Time(int timeVal, TimeUnit granularity, Extractor<Long, DATA> timeToData) {
+ private Time(int timeVal, TimeUnit timeUnit) {
this.timeVal = timeVal;
- this.granularity = granularity;
- this.timeToData = timeToData;
- }
-
- /**
- * Creates an helper representing a trigger which triggers every given
- * timeVal or an eviction which evicts all elements older than timeVal.
- *
- * The default granularity for timeVal used in this method is seconds.
- *
- * @param timeVal
- * The number of time units measured in seconds.
- * @param timeToData
- * This policy creates fake elements to not miss windows in case
- * no element arrived within the duration of the window. This
- * extractor should wrap a long into such an element of type
- * DATA.
- */
- public Time(int timeVal, Extractor<Long, DATA> timeToData) {
- this(timeVal, TimeUnit.SECONDS, timeToData);
+ this.granularity = timeUnit;
+ this.longToDATAExtractor = new NullExtractor<DATA>();
+ this.timeStamp = new DefaultTimeStamp<DATA>();
+ this.delay = 0;
}
@Override
public EvictionPolicy<DATA> toEvict() {
- return new TimeEvictionPolicy<DATA>(granularityInMillis(), new DefaultTimeStamp<DATA>());
+ return new TimeEvictionPolicy<DATA>(granularityInMillis(), timeStamp);
}
@Override
public TriggerPolicy<DATA> toTrigger() {
- return new TimeTriggerPolicy<DATA>(granularityInMillis(), new DefaultTimeStamp<DATA>(),
- timeToData);
+ return new TimeTriggerPolicy<DATA>(granularityInMillis(), timeStamp, delay,
+ longToDATAExtractor);
}
/**
@@ -104,22 +86,39 @@ public class Time<DATA> implements WindowingHelper<DATA> {
* the smallest possible granularity is milliseconds. Any smaller
* time unit might cause an error at runtime due to conversion
* problems.
- * @param timeToData
- * This policy creates fake elements to not miss windows in case
- * no element arrived within the duration of the window. This
- * extractor should wrap a long into such an element of type
- * DATA.
* @return an helper representing a trigger which triggers every given
* timeVal or an eviction which evicts all elements older than
* timeVal.
*/
- public static <DATA> Time<DATA> of(int timeVal, TimeUnit granularity,
- Extractor<Long, DATA> timeToData) {
- return new Time<DATA>(timeVal, granularity, timeToData);
+ public static <DATA> Time<DATA> of(int timeVal, TimeUnit granularity) {
+ return new Time<DATA>(timeVal, granularity);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <R> Time<R> withTimeStamp(TimeStamp<R> timeStamp, Extractor<Long, R> extractor) {
+ this.timeStamp = (TimeStamp<DATA>) timeStamp;
+ this.longToDATAExtractor = (Extractor<Long, DATA>) extractor;
+ return (Time<R>) this;
+ }
+
+ public Time<DATA> withDelay(long delay) {
+ this.delay = delay;
+ return this;
}
private long granularityInMillis() {
return this.granularity.toMillis(this.timeVal);
}
+ public static class NullExtractor<T> implements Extractor<Long, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public T extract(Long in) {
+ return null;
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
index d5ae932..7a3c75a 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CountTriggerPolicy.java
@@ -30,7 +30,7 @@ public class CountTriggerPolicy<IN> implements CloneableTriggerPolicy<IN> {
*/
private static final long serialVersionUID = -6357200688886103968L;
- private static final int DEFAULT_START_VALUE = 0;
+ public static final int DEFAULT_START_VALUE = 0;
private int counter;
private int max;
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
index d9acc84..b212293 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeEvictionPolicy.java
@@ -87,10 +87,10 @@ public class TimeEvictionPolicy<DATA> implements ActiveEvictionPolicy<DATA>,
// delete and count expired tuples
int counter = 0;
+ long threshold = timestamp.getTimestamp(datapoint) - granularity;
while (!buffer.isEmpty()) {
- if (timestamp.getTimestamp(buffer.getFirst()) < timestamp.getTimestamp(datapoint)
- - granularity) {
+ if (timestamp.getTimestamp(buffer.getFirst()) < threshold) {
buffer.removeFirst();
counter++;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
index ff93ac9..1dd713f 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -41,9 +41,11 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
*/
private static final long serialVersionUID = -5122753802440196719L;
- private long startTime;
- private long granularity;
- private TimeStamp<DATA> timestamp;
+ protected long startTime;
+ protected long granularity;
+ protected TimeStamp<DATA> timestamp;
+ protected long delay;
+
private Extractor<Long, DATA> longToDATAExtractor;
/**
@@ -54,8 +56,8 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
* example, the policy will trigger at every second point in time.
*
* @param granularity
- * The granularity of the trigger. If this value is set to 2 the
- * policy will trigger at every second time point
+ * The granularity of the trigger. If this value is set to x the
+ * policy will trigger at every x-th time point
* @param timestamp
* The {@link TimeStamp} to measure the time with. This can be
* either user defined of provided by the API.
@@ -72,10 +74,10 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
/**
* This is mostly the same as
- * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, TimeStamp)}. In addition
- * to granularity and timestamp a delay can be specified for the first
- * trigger. If the start time given by the timestamp is x, the delay is y,
- * and the granularity is z, the first trigger will happen at x+y+z.
+ * {@link TimeTriggerPolicy#TimeTriggerPolicy(long, TimeStamp)}. In
+ * addition to granularity and timestamp a delay can be specified for the
+ * first trigger. If the start time given by the timestamp is x, the delay
+ * is y, and the granularity is z, the first trigger will happen at x+y+z.
*
* @param granularity
* The granularity of the trigger. If this value is set to 2 the
@@ -98,21 +100,9 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
this.startTime = timestamp.getStartTime() + delay;
this.timestamp = timestamp;
this.granularity = granularity;
+ this.delay = delay;
this.longToDATAExtractor = timeWrapper;
- }
- @Override
- public synchronized boolean notifyTrigger(DATA datapoint) {
- long recordTime = timestamp.getTimestamp(datapoint);
- // start time is included, but end time is excluded: >=
- if (recordTime >= startTime + granularity) {
- if (granularity != 0) {
- startTime = recordTime - ((recordTime - startTime) % granularity);
- }
- return true;
- } else {
- return false;
- }
}
/**
@@ -126,7 +116,8 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
// check if there is more then one window border missed
// use > here. In case >= would fit, the regular call will do the job.
while (timestamp.getTimestamp(datapoint) > startTime + granularity) {
- fakeElements.add(longToDATAExtractor.extract(startTime += granularity));
+ startTime += granularity;
+ fakeElements.add(longToDATAExtractor.extract(startTime));
}
return (DATA[]) fakeElements.toArray();
}
@@ -162,7 +153,7 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
if (System.currentTimeMillis() >= startTime + granularity) {
startTime += granularity;
- callback.sendFakeElement(longToDATAExtractor.extract(startTime += granularity));
+ callback.sendFakeElement(longToDATAExtractor.extract(startTime));
}
}
@@ -192,8 +183,22 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
}
@Override
+ public synchronized boolean notifyTrigger(DATA datapoint) {
+ long recordTime = timestamp.getTimestamp(datapoint);
+ // start time is included, but end time is excluded: >=
+ if (recordTime >= startTime + granularity) {
+ if (granularity != 0) {
+ startTime = recordTime - ((recordTime - startTime) % granularity);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
public TimeTriggerPolicy<DATA> clone() {
- return new TimeTriggerPolicy<DATA>(granularity, timestamp, 0, longToDATAExtractor);
+ return new TimeTriggerPolicy<DATA>(granularity, timestamp, delay, longToDATAExtractor);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
index d65436c..c494d5f 100755
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
@@ -17,13 +17,25 @@
package org.apache.flink.streaming.api;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
import org.apache.flink.api.common.functions.FilterFunction;
+import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.api.windowing.helper.Time;
+import org.apache.flink.util.Collector;
import org.junit.Test;
-public class PrintTest{
+public class PrintTest implements Serializable {
private static final long MEMORYSIZE = 32;
@@ -44,12 +56,52 @@ public class PrintTest{
return true;
}
}
-
+
@Test
public void test() throws Exception {
LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
- env.generateSequence(1, 10).map(new IdentityMap()).filter(new FilterAll()).print();
+
+ List<Tuple2<String, Integer>> input = new ArrayList<Tuple2<String, Integer>>();
+
+ env.fromElements(1, 2, 3, 4, 5, 6, 7, 8, 9)
+ .window(Time.of(2, TimeUnit.MILLISECONDS).withTimeStamp(new TimeStamp<Integer>() {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Integer value) {
+
+ return value;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 1;
+ }
+ }, new Extractor<Long, Integer>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer extract(Long in) {
+ return in.intValue();
+ }
+ })).every(Count.of(2)).reduceGroup(new GroupReduceFunction<Integer, String>() {
+
+ @Override
+ public void reduce(Iterable<Integer> values, Collector<String> out)
+ throws Exception {
+ String o = "|";
+ for (Integer v : values) {
+ o = o + v + "|";
+ }
+ out.collect(o);
+ }
+ }).print();
env.executeTest(MEMORYSIZE);
-
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceTest.java
deleted file mode 100644
index f157a08..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchGroupReduceTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class BatchGroupReduceTest {
-
- public static final class MySlidingBatchReduce implements GroupReduceFunction<Integer, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<Integer> values, Collector<String> out) throws Exception {
- for (Integer value : values) {
- out.collect(value.toString());
- }
- out.collect(END_OF_BATCH);
- }
- }
-
- private final static String END_OF_BATCH = "end of batch";
- private final static int SLIDING_BATCH_SIZE = 3;
- private final static int SLIDE_SIZE = 2;
-
- @Test
- public void slidingBatchReduceTest() {
- BatchGroupReduceInvokable<Integer, String> invokable = new BatchGroupReduceInvokable<Integer, String>(
- new MySlidingBatchReduce(), SLIDING_BATCH_SIZE, SLIDE_SIZE);
-
- List<String> expected = Arrays.asList("1", "2", "3", END_OF_BATCH, "3", "4", "5",
- END_OF_BATCH, "5", "6", "7", END_OF_BATCH);
- List<String> actual = MockInvokable.createAndExecute(invokable,
- Arrays.asList(1, 2, 3, 4, 5, 6, 7));
-
- assertEquals(expected, actual);
- }
-
- public static final class MyBatchReduce implements GroupReduceFunction<Double, Double> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<Double> values, Collector<Double> out) throws Exception {
-
- Double sum = 0.;
- Double count = 0.;
- for (Double value : values) {
- sum += value;
- count++;
- }
- if (count > 0) {
- out.collect(Double.valueOf(sum / count));
- }
- }
- }
-
- private static final int BATCH_SIZE = 5;
-
- @Test
- public void nonSlidingBatchReduceTest() {
- List<Double> inputs = new ArrayList<Double>();
- for (Double i = 1.; i <= 100; i++) {
- inputs.add(i);
- }
-
- BatchGroupReduceInvokable<Double, Double> invokable = new BatchGroupReduceInvokable<Double, Double>(
- new MyBatchReduce(), BATCH_SIZE, BATCH_SIZE);
-
- List<Double> avgs = MockInvokable.createAndExecute(invokable, inputs);
-
- for (int i = 0; i < avgs.size(); i++) {
- assertEquals(3.0 + i * BATCH_SIZE, avgs.get(i), 0);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceTest.java
deleted file mode 100755
index b154f7f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/BatchReduceTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.junit.Test;
-
-public class BatchReduceTest {
-
- @Test
- public void BatchReduceInvokableTest() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- for (Integer i = 1; i <= 10; i++) {
- inputs.add(i);
- }
- BatchReduceInvokable<Integer> invokable = new BatchReduceInvokable<Integer>(
- new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- }, 3, 2);
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(6);
- expected.add(12);
- expected.add(18);
- expected.add(24);
- expected.add(19);
- assertEquals(expected, MockInvokable.createAndExecute(invokable, inputs));
-
- List<Integer> inputs2 = new ArrayList<Integer>();
- inputs2.add(1);
- inputs2.add(2);
- inputs2.add(-1);
- inputs2.add(-3);
- inputs2.add(-4);
-
- BatchReduceInvokable<Integer> invokable2 = new BatchReduceInvokable<Integer>(
- new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- if (value1 <= value2) {
- return value1;
- } else {
- return value2;
- }
- }
- }, 2, 3);
-
- List<Integer> expected2 = new ArrayList<Integer>();
- expected2.add(1);
- expected2.add(-4);
-
- assertEquals(expected2, MockInvokable.createAndExecute(invokable2, inputs2));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceTest.java
deleted file mode 100755
index db2b8cf..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchGroupReduceTest.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.apache.flink.streaming.util.keys.ObjectKeySelector;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class GroupedBatchGroupReduceTest {
-
- public static final class MySlidingBatchReduce1 implements GroupReduceFunction<Integer, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<Integer> values, Collector<String> out) throws Exception {
- for (Integer value : values) {
- out.collect(value.toString());
- }
- out.collect(END_OF_GROUP);
- }
- }
-
- public static final class MySlidingBatchReduce2 extends
- RichGroupReduceFunction<Tuple2<Integer, String>, String> {
- private static final long serialVersionUID = 1L;
-
- String openString;
-
- @Override
- public void reduce(Iterable<Tuple2<Integer, String>> values, Collector<String> out)
- throws Exception {
- out.collect(openString);
- for (Tuple2<Integer, String> value : values) {
- out.collect(value.f0.toString());
- }
- out.collect(END_OF_GROUP);
- }
-
- @Override
- public void open(Configuration c) {
- openString = "open";
- }
- }
-
- private final static String END_OF_GROUP = "end of group";
-
- @SuppressWarnings("unchecked")
- @Test
- public void slidingBatchGroupReduceTest() {
- @SuppressWarnings("rawtypes")
- GroupedBatchGroupReduceInvokable<Integer, String> invokable1 = new GroupedBatchGroupReduceInvokable<Integer, String>(
- new MySlidingBatchReduce1(), 2, 2, new ObjectKeySelector());
-
- List<String> expected = Arrays.asList("1", "1", END_OF_GROUP, "3", "3", END_OF_GROUP, "2",
- END_OF_GROUP);
- List<String> actual = MockInvokable.createAndExecute(invokable1,
- Arrays.asList(1, 1, 2, 3, 3));
-
- assertEquals(expected, actual);
-
- GroupedBatchGroupReduceInvokable<Tuple2<Integer, String>, String> invokable2 = new GroupedBatchGroupReduceInvokable<Tuple2<Integer, String>, String>(
- new MySlidingBatchReduce2(), 2, 2, new TupleKeySelector<Tuple2<Integer, String>>(1));
-
- expected = Arrays.asList("open", "1", "2", END_OF_GROUP, "open", "3", "3", END_OF_GROUP,
- "open", "4", END_OF_GROUP);
- actual = MockInvokable.createAndExecute(invokable2, Arrays.asList(
- new Tuple2<Integer, String>(1, "a"), new Tuple2<Integer, String>(2, "a"),
- new Tuple2<Integer, String>(3, "b"), new Tuple2<Integer, String>(3, "b"),
- new Tuple2<Integer, String>(4, "a")));
-
- assertEquals(expected, actual);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceTest.java
deleted file mode 100755
index 783119c..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedBatchReduceTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.apache.flink.streaming.util.keys.ObjectKeySelector;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
-import org.junit.Test;
-
-public class GroupedBatchReduceTest {
-
- @Test
- public void BatchReduceInvokableTest() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(1);
- inputs.add(5);
- inputs.add(5);
- inputs.add(5);
- inputs.add(1);
- inputs.add(1);
- inputs.add(5);
- inputs.add(1);
- inputs.add(5);
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(15);
- expected.add(3);
- expected.add(3);
- expected.add(15);
-
- GroupedBatchReduceInvokable<Integer> invokable = new GroupedBatchReduceInvokable<Integer>(
- new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- }, 3, 2, new ObjectKeySelector<Integer>());
-
- List<Integer> actual = MockInvokable.createAndExecute(invokable, inputs);
- assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(actual));
- assertEquals(expected.size(), actual.size());
-
- List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
- inputs2.add(new Tuple2<Integer, String>(1, "a"));
- inputs2.add(new Tuple2<Integer, String>(0, "b"));
- inputs2.add(new Tuple2<Integer, String>(2, "a"));
- inputs2.add(new Tuple2<Integer, String>(-1, "a"));
- inputs2.add(new Tuple2<Integer, String>(-2, "a"));
- inputs2.add(new Tuple2<Integer, String>(10, "a"));
- inputs2.add(new Tuple2<Integer, String>(2, "b"));
- inputs2.add(new Tuple2<Integer, String>(1, "a"));
-
- List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
- expected2.add(new Tuple2<Integer, String>(-1, "a"));
- expected2.add(new Tuple2<Integer, String>(-2, "a"));
- expected2.add(new Tuple2<Integer, String>(0, "b"));
-
- GroupedBatchReduceInvokable<Tuple2<Integer, String>> invokable2 = new GroupedBatchReduceInvokable<Tuple2<Integer, String>>(
- new ReduceFunction<Tuple2<Integer, String>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
- Tuple2<Integer, String> value2) throws Exception {
- if (value1.f0 <= value2.f0) {
- return value1;
- } else {
- return value2;
- }
- }
- }, 3, 3, new TupleKeySelector<Tuple2<Integer, String>>(1));
-
- List<Tuple2<Integer, String>> actual2 = MockInvokable.createAndExecute(invokable2, inputs2);
-
- assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
- new HashSet<Tuple2<Integer, String>>(actual2));
- assertEquals(expected2.size(), actual2.size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokableTest.java
deleted file mode 100644
index e93647f..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowGroupReduceInvokableTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class GroupedWindowGroupReduceInvokableTest {
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- @Test
- public void windowReduceTest() {
- List<Tuple2<String, Integer>> inputs2 = new ArrayList<Tuple2<String, Integer>>();
- inputs2.add(new Tuple2<String, Integer>("a", 1));
- inputs2.add(new Tuple2<String, Integer>("a", 2));
- inputs2.add(new Tuple2<String, Integer>("b", 2));
- inputs2.add(new Tuple2<String, Integer>("b", 2));
- inputs2.add(new Tuple2<String, Integer>("b", 5));
- inputs2.add(new Tuple2<String, Integer>("a", 7));
- inputs2.add(new Tuple2<String, Integer>("b", 9));
- inputs2.add(new Tuple2<String, Integer>("b", 10));
- // 1,2-4,5-7,8-10
-
- List<Tuple2<String, Integer>> expected2 = new ArrayList<Tuple2<String, Integer>>();
- expected2.add(new Tuple2<String, Integer>("a", 3));
- expected2.add(new Tuple2<String, Integer>("b", 4));
- expected2.add(new Tuple2<String, Integer>("b", 5));
- expected2.add(new Tuple2<String, Integer>("a", 7));
- expected2.add(new Tuple2<String, Integer>("b", 10));
-
- GroupedWindowGroupReduceInvokable<Tuple2<String, Integer>, Tuple2<String, Integer>> invokable2 = new GroupedWindowGroupReduceInvokable<Tuple2<String, Integer>, Tuple2<String, Integer>>(
- new GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<Tuple2<String, Integer>> values,
- Collector<Tuple2<String, Integer>> out) throws Exception {
- Tuple2<String, Integer> outTuple = new Tuple2<String, Integer>("", 0);
-
- for (@SuppressWarnings("unused")
- Tuple2<String, Integer> value : values) {
- }
-
- for (Tuple2<String, Integer> value : values) {
- outTuple.f0 = value.f0;
- outTuple.f1 += value.f1;
- }
- out.collect(outTuple);
- }
- }, 2, 3, new TupleKeySelector( 0),
- new TimeStamp<Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Tuple2<String, Integer> value) {
- return value.f1;
- }
-
- @Override
- public long getStartTime() {
- return 1;
- }
- });
-
- List<Tuple2<String, Integer>> actual2 = MockInvokable.createAndExecute(invokable2, inputs2);
-
- assertEquals(new HashSet<Tuple2<String, Integer>>(expected2),
- new HashSet<Tuple2<String, Integer>>(actual2));
- assertEquals(expected2.size(), actual2.size());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
index 93848db..ebfed05 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokableTest.java
@@ -17,7 +17,7 @@
package org.apache.flink.streaming.api.invokable.operator;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.HashSet;
@@ -30,12 +30,12 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.windowing.extractor.Extractor;
import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
import org.apache.flink.streaming.util.MockInvokable;
@@ -78,7 +78,7 @@ public class GroupedWindowingInvokableTest {
LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
- GroupedWindowingInvokable<Integer> invokable = new GroupedWindowingInvokable<Integer>(
+ GroupedWindowingInvokable<Integer, Integer> invokable = new GroupedWindowingInvokable<Integer, Integer>(
new ReduceFunction<Integer>() {
private static final long serialVersionUID = 1L;
@@ -95,11 +95,11 @@ public class GroupedWindowingInvokableTest {
}
}, triggers, evictions, centralTriggers);
- List<Tuple2<Integer, String[]>> result = MockInvokable.createAndExecute(invokable, inputs);
+ List<Integer> result = MockInvokable.createAndExecute(invokable, inputs);
List<Integer> actual = new LinkedList<Integer>();
- for (Tuple2<Integer, String[]> current : result) {
- actual.add(current.f0);
+ for (Integer current : result) {
+ actual.add(current);
}
assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(actual));
@@ -137,7 +137,7 @@ public class GroupedWindowingInvokableTest {
LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
- GroupedWindowingInvokable<Tuple2<Integer, String>> invokable2 = new GroupedWindowingInvokable<Tuple2<Integer, String>>(
+ GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
new ReduceFunction<Tuple2<Integer, String>>() {
private static final long serialVersionUID = 1L;
@@ -153,12 +153,11 @@ public class GroupedWindowingInvokableTest {
}, new TupleKeySelector<Tuple2<Integer, String>>(1), triggers, evictions,
centralTriggers);
- List<Tuple2<Tuple2<Integer, String>, String[]>> result = MockInvokable.createAndExecute(
- invokable2, inputs2);
+ List<Tuple2<Integer, String>> result = MockInvokable.createAndExecute(invokable2, inputs2);
List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
- for (Tuple2<Tuple2<Integer, String>, String[]> current : result) {
- actual2.add(current.f0);
+ for (Tuple2<Integer, String> current : result) {
+ actual2.add(current);
}
assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
@@ -266,14 +265,13 @@ public class GroupedWindowingInvokableTest {
LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
- GroupedWindowingInvokable<Tuple2<Integer, String>> invokable = new GroupedWindowingInvokable<Tuple2<Integer, String>>(
+ GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowingInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
myReduceFunction, new TupleKeySelector<Tuple2<Integer, String>>(1),
distributedTriggers, evictions, triggers);
ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
- for (Tuple2<Tuple2<Integer, String>, String[]> t : MockInvokable.createAndExecute(
- invokable, inputs)) {
- result.add(t.f0);
+ for (Tuple2<Integer, String> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
}
assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
@@ -330,7 +328,7 @@ public class GroupedWindowingInvokableTest {
}
};
- GroupedWindowingInvokable<Integer> invokable = new GroupedWindowingInvokable<Integer>(
+ GroupedWindowingInvokable<Integer, Integer> invokable = new GroupedWindowingInvokable<Integer, Integer>(
myReduceFunction, new KeySelector<Integer, Integer>() {
private static final long serialVersionUID = 1L;
@@ -341,8 +339,8 @@ public class GroupedWindowingInvokableTest {
}, distributedTriggers, evictions, triggers);
ArrayList<Integer> result = new ArrayList<Integer>();
- for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t.f0);
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
}
assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
@@ -350,8 +348,8 @@ public class GroupedWindowingInvokableTest {
}
/**
- * Test for combination of centralized trigger and
- * distributed trigger at the same time
+ * Test for combination of centralized trigger and distributed trigger at
+ * the same time
*/
@Test
public void testGroupedWindowingInvokableCentralAndDistrTrigger() {
@@ -406,7 +404,7 @@ public class GroupedWindowingInvokableTest {
}
};
- GroupedWindowingInvokable<Integer> invokable = new GroupedWindowingInvokable<Integer>(
+ GroupedWindowingInvokable<Integer, Integer> invokable = new GroupedWindowingInvokable<Integer, Integer>(
myReduceFunction, new KeySelector<Integer, Integer>() {
private static final long serialVersionUID = 1L;
@@ -417,8 +415,8 @@ public class GroupedWindowingInvokableTest {
}, distributedTriggers, evictions, triggers);
ArrayList<Integer> result = new ArrayList<Integer>();
- for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t.f0);
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
}
assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokableTest.java
deleted file mode 100644
index 3e3179a..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokableTest.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.apache.flink.util.Collector;
-import org.junit.Before;
-import org.junit.Test;
-
-public class WindowGroupReduceInvokableTest {
-
- public static final class MySlidingWindowReduce implements GroupReduceFunction<Integer, String> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public void reduce(Iterable<Integer> values, Collector<String> out) throws Exception {
- for (Integer value : values) {
- out.collect(value.toString());
- }
- out.collect(EOW);
- }
- }
-
- public static final class MyTimestamp implements TimeStamp<Integer> {
- private static final long serialVersionUID = 1L;
-
- private Iterator<Long> timestamps;
- private long start;
-
- public MyTimestamp(List<Long> timestamps) {
- this.timestamps = timestamps.iterator();
- this.start = timestamps.get(0);
- }
-
- @Override
- public long getTimestamp(Integer value) {
- long ts = timestamps.next();
- return ts;
- }
-
- @Override
- public long getStartTime() {
- return start;
- }
- }
-
- private final static String EOW = "|";
-
- private static List<WindowGroupReduceInvokable<Integer, String>> invokables = new ArrayList<WindowGroupReduceInvokable<Integer, String>>();
- private static List<List<String>> expectedResults = new ArrayList<List<String>>();
-
- @Before
- public void before() {
- long windowSize = 3;
- long slideSize = 2;
- List<Long> timestamps = Arrays.asList(101L, 102L, 103L, 104L, 105L, 106L, 107L, 108L, 109L,
- 110L);
- expectedResults.add(Arrays.asList("1", "2", "3", EOW, "3", "4", "5", EOW, "5", "6", "7",
- EOW, "7", "8", "9", EOW, "9", "10", EOW));
- invokables.add(new WindowGroupReduceInvokable<Integer, String>(new MySlidingWindowReduce(),
- windowSize, slideSize, new MyTimestamp(timestamps)));
-
- windowSize = 10;
- slideSize = 5;
- timestamps = Arrays.asList(101L, 103L, 121L, 122L, 123L, 124L, 180L, 181L, 185L, 190L);
- expectedResults.add(Arrays.asList("1", "2", EOW, "3", "4", "5", "6", EOW, "3",
- "4", "5", "6", EOW, "7", EOW, "7",
- "8", "9", EOW, "8", "9", "10", EOW));
- invokables.add(new WindowGroupReduceInvokable<Integer, String>(new MySlidingWindowReduce(),
- windowSize, slideSize, new MyTimestamp(timestamps)));
-
- windowSize = 10;
- slideSize = 4;
- timestamps = Arrays.asList(101L, 103L, 110L, 112L, 113L, 114L, 120L, 121L, 125L, 130L);
- expectedResults.add(Arrays.asList("1", "2","3" ,EOW, "3", "4", "5","6", EOW, "3", "4", "5", "6",
- EOW, "5", "6", "7", "8", EOW, "7", "8", "9", EOW, "8","9",
- "10", EOW));
- invokables.add(new WindowGroupReduceInvokable<Integer, String>(new MySlidingWindowReduce(),
- windowSize, slideSize, new MyTimestamp(timestamps)));
- }
-
- @Test
- public void slidingBatchReduceTest() {
- List<List<String>> actualResults = new ArrayList<List<String>>();
-
- for (WindowGroupReduceInvokable<Integer, String> invokable : invokables) {
- List<String> result = MockInvokable.createAndExecute(invokable,
- Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10));
- actualResults.add(result);
- }
-
- Iterator<List<String>> actualResult = actualResults.iterator();
-
- for (List<String> expectedResult : expectedResults) {
- assertEquals(expectedResult, actualResult.next());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokableTest.java
deleted file mode 100755
index 5d10eff..0000000
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokableTest.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.invokable.util.TimeStamp;
-import org.apache.flink.streaming.util.MockInvokable;
-import org.apache.flink.streaming.util.keys.TupleKeySelector;
-import org.junit.Test;
-
-public class WindowReduceInvokableTest {
-
- @Test
- public void windowReduceTest() {
-
- List<Integer> inputs = new ArrayList<Integer>();
- inputs.add(1);
- inputs.add(2);
- inputs.add(2);
- inputs.add(3);
- inputs.add(4);
- inputs.add(5);
- inputs.add(10);
- inputs.add(11);
- inputs.add(11);
- // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
- // 12-12-5-10-32
-
- List<Integer> expected = new ArrayList<Integer>();
- expected.add(12);
- expected.add(12);
- expected.add(5);
- expected.add(10);
- expected.add(32);
-
- WindowReduceInvokable<Integer> invokable = new WindowReduceInvokable<Integer>(
- new ReduceFunction<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer reduce(Integer value1, Integer value2) throws Exception {
- return value1 + value2;
- }
- }, 4, 2, new TimeStamp<Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Integer value) {
- return value;
- }
-
- @Override
- public long getStartTime() {
- return 1;
- }
- });
-
- assertEquals(expected, MockInvokable.createAndExecute(invokable, inputs));
-
- List<Tuple2<String, Integer>> inputs2 = new ArrayList<Tuple2<String, Integer>>();
- inputs2.add(new Tuple2<String, Integer>("a", 1));
- inputs2.add(new Tuple2<String, Integer>("a", 2));
- inputs2.add(new Tuple2<String, Integer>("b", 2));
- inputs2.add(new Tuple2<String, Integer>("b", 2));
- inputs2.add(new Tuple2<String, Integer>("b", 5));
- inputs2.add(new Tuple2<String, Integer>("a", 7));
- inputs2.add(new Tuple2<String, Integer>("b", 9));
- inputs2.add(new Tuple2<String, Integer>("b", 10));
-
- List<Tuple2<String, Integer>> expected2 = new ArrayList<Tuple2<String, Integer>>();
- expected2.add(new Tuple2<String, Integer>("a", 3));
- expected2.add(new Tuple2<String, Integer>("b", 4));
- expected2.add(new Tuple2<String, Integer>("b", 5));
- expected2.add(new Tuple2<String, Integer>("a", 7));
- expected2.add(new Tuple2<String, Integer>("b", 10));
-
- @SuppressWarnings({ "unchecked", "rawtypes" })
- GroupedWindowReduceInvokable<Tuple2<String, Integer>> invokable2 = new GroupedWindowReduceInvokable<Tuple2<String, Integer>>(
- new ReduceFunction<Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1,
- Tuple2<String, Integer> value2) throws Exception {
- return new Tuple2<String, Integer>(value1.f0, value1.f1 + value2.f1);
- }
- }, 2, 3, new TupleKeySelector(0), new TimeStamp<Tuple2<String, Integer>>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getTimestamp(Tuple2<String, Integer> value) {
- return value.f1;
- }
-
- @Override
- public long getStartTime() {
- return 1;
- }
- });
-
- List<Tuple2<String, Integer>> actual2 = MockInvokable.createAndExecute(invokable2, inputs2);
- assertEquals(new HashSet<Tuple2<String, Integer>>(expected2),
- new HashSet<Tuple2<String, Integer>>(actual2));
- assertEquals(expected2.size(), actual2.size());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
index 79df1c5..8e96a7c 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
@@ -17,21 +17,20 @@
package org.apache.flink.streaming.api.invokable.operator;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
import org.apache.flink.streaming.util.MockInvokable;
import org.junit.Test;
@@ -92,6 +91,7 @@ public class WindowingInvokableTest {
// trigger after 4, then every 2)
triggers.add(new TimeTriggerPolicy<Integer>(2L, myTimeStamp, 2L,
new Extractor<Long, Integer>() {
+
private static final long serialVersionUID = 1L;
@Override
@@ -103,12 +103,12 @@ public class WindowingInvokableTest {
// Always delete all elements older then 4
evictions.add(new TimeEvictionPolicy<Integer>(4L, myTimeStamp));
- WindowingInvokable<Integer> invokable = new WindowingInvokable<Integer>(myReduceFunction,
- triggers, evictions);
+ WindowingInvokable<Integer, Integer> invokable = new WindowingReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
ArrayList<Integer> result = new ArrayList<Integer>();
- for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t.f0);
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
}
assertEquals(expected, result);
@@ -147,8 +147,8 @@ public class WindowingInvokableTest {
// time after the 3rd element
evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
- WindowingInvokable<Integer> invokable = new WindowingInvokable<Integer>(myReduceFunction,
- triggers, evictions);
+ WindowingInvokable<Integer, Integer> invokable = new WindowingReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
List<Integer> expected = new ArrayList<Integer>();
expected.add(6);
@@ -157,8 +157,8 @@ public class WindowingInvokableTest {
expected.add(24);
expected.add(19);
List<Integer> result = new ArrayList<Integer>();
- for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t.f0);
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
}
assertEquals(expected, result);
@@ -200,16 +200,16 @@ public class WindowingInvokableTest {
// time after on the 5th element
evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
- WindowingInvokable<Integer> invokable2 = new WindowingInvokable<Integer>(myReduceFunction,
- triggers, evictions);
+ WindowingInvokable<Integer, Integer> invokable2 = new WindowingReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
List<Integer> expected2 = new ArrayList<Integer>();
expected2.add(1);
expected2.add(-4);
result = new ArrayList<Integer>();
- for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable2, inputs2)) {
- result.add(t.f0);
+ for (Integer t : MockInvokable.createAndExecute(invokable2, inputs2)) {
+ result.add(t);
}
assertEquals(expected2, result);
@@ -223,8 +223,8 @@ public class WindowingInvokableTest {
triggers.add(new CountTriggerPolicy<Integer>(3));
LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
- evictions.add(new CountEvictionPolicy<Integer>(2,2));
- evictions.add(new CountEvictionPolicy<Integer>(3,3));
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2));
+ evictions.add(new CountEvictionPolicy<Integer>(3, 3));
List<Integer> inputs = new ArrayList<Integer>();
for (Integer i = 1; i <= 10; i++) {
@@ -258,12 +258,12 @@ public class WindowingInvokableTest {
}
};
- WindowingInvokable<Integer> invokable = new WindowingInvokable<Integer>(myReduceFunction,
- triggers, evictions);
+ WindowingInvokable<Integer, Integer> invokable = new WindowingReduceInvokable<Integer>(
+ myReduceFunction, triggers, evictions);
ArrayList<Integer> result = new ArrayList<Integer>();
- for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
- result.add(t.f0);
+ for (Integer t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t);
}
assertEquals(expected, result);
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
index 839564d..34e292a 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicyTest.java
@@ -17,12 +17,15 @@
package org.apache.flink.streaming.api.windowing.policy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
import org.apache.flink.streaming.api.invokable.util.TimeStamp;
import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.helper.Time;
import org.junit.Test;
-import static org.junit.Assert.*;
-
public class TimeTriggerPolicyTest {
@Test
@@ -49,15 +52,8 @@ public class TimeTriggerPolicyTest {
// test different granularity
for (long granularity = 0; granularity < 31; granularity++) {
// create policy
- TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity, timeStamp,
- new Extractor<Long, Integer>() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public Integer extract(Long in) {
- return in.intValue();
- }
- });
+ TriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(granularity,
+ timeStamp, new Time.NullExtractor<Integer>());
// remember window border
// Remark: This might NOT work in case the timeStamp uses
@@ -88,7 +84,7 @@ public class TimeTriggerPolicyTest {
@Test
public void timeTriggerPreNotifyTest() {
// create some test data
- Integer[] times = { 1, 3, 20, 26};
+ Integer[] times = { 1, 3, 20, 26 };
// create a timestamp
@SuppressWarnings("serial")
@@ -107,8 +103,9 @@ public class TimeTriggerPolicyTest {
};
// create policy
- TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5, timeStamp,
- new Extractor<Long, Integer>() {
+ TimeTriggerPolicy<Integer> policy = new TimeTriggerPolicy<Integer>(5,
+ timeStamp, new Extractor<Long, Integer>() {
+
private static final long serialVersionUID = 1L;
@Override
@@ -117,16 +114,16 @@ public class TimeTriggerPolicyTest {
}
});
- //expected result
- Integer[][] result={{},{},{5,10,15},{25}};
-
- //call policy
- for (int i=0;i<times.length;i++){
- arrayEqualityCheck(result[i],policy.preNotifyTrigger(times[i]));
+ // expected result
+ Integer[][] result = { {}, {}, { 5, 10, 15 }, { 25 } };
+
+ // call policy
+ for (int i = 0; i < times.length; i++) {
+ arrayEqualityCheck(result[i], policy.preNotifyTrigger(times[i]));
policy.notifyTrigger(times[i]);
}
}
-
+
private void arrayEqualityCheck(Object[] array1, Object[] array2) {
assertEquals("The result arrays must have the same length", array1.length, array2.length);
for (int i = 0; i < array1.length; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
index 2def142..375c86d 100755
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
+++ b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
@@ -17,11 +17,14 @@
package org.apache.flink.streaming.examples.ml;
+import java.util.concurrent.TimeUnit;
+
import org.apache.flink.api.common.functions.GroupReduceFunction;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.function.co.CoMapFunction;
import org.apache.flink.streaming.api.function.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Time;
import org.apache.flink.util.Collector;
/**
@@ -58,7 +61,8 @@ public class IncrementalLearningSkeleton {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// build new model on every second of new data
- DataStream<Double[]> model = env.addSource(new TrainingDataSource()).window(5000)
+ DataStream<Double[]> model = env.addSource(new TrainingDataSource())
+ .window(Time.of(5000, TimeUnit.MILLISECONDS))
.reduceGroup(new PartialModelBuilder());
// use partial model for prediction
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f96ba06e/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java b/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java
deleted file mode 100644
index c0e768b..0000000
--- a/flink-addons/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/BasicExample.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.windowing;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.source.SourceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Count;
-import org.apache.flink.util.Collector;
-
-/**
- * A minimal example as introduction to the policy based windowing
- */
-public class BasicExample {
-
- private static final int PARALLELISM = 1;
-
- public static void main(String[] args) throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment
- .createLocalEnvironment(PARALLELISM);
-
- // This reduce function does a String concat.
- ReduceFunction<String> reduceFunction = new ReduceFunction<String>() {
-
- /**
- * Auto generates version ID
- */
- private static final long serialVersionUID = 1L;
-
- @Override
- public String reduce(String value1, String value2) throws Exception {
- return value1 + "|" + value2;
- }
-
- };
-
- DataStream<Tuple2<String, String[]>> stream = env.addSource(new BasicSource())
- .window(Count.of(5)).every(Count.of(2)).reduce(reduceFunction);
-
- stream.print();
-
- env.execute();
- }
-
- public static class BasicSource implements SourceFunction<String> {
-
- private static final long serialVersionUID = 1L;
- String str = new String("streaming");
-
- @Override
- public void invoke(Collector<String> out) throws Exception {
- // continuous emit
- while (true) {
- out.collect(str);
- }
- }
- }
-
-}
[07/34] incubator-flink git commit: [streaming] Created trigger and
eviction policies based on data punctuation. (incl. test cases)
Posted by mb...@apache.org.
[streaming] Created trigger and eviction policies based on data punctuation. (incl. test cases)
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/a6cc8afb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/a6cc8afb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/a6cc8afb
Branch: refs/heads/master
Commit: a6cc8afb89f61d6067f4ccf85b50a5899630346a
Parents: 05c1519
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:44:02 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../api/windowing/policy/PunctuationPolicy.java | 110 +++++++++++++++++
.../windowing/policy/PunctuationPolicyTest.java | 123 +++++++++++++++++++
2 files changed, 233 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a6cc8afb/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
new file mode 100644
index 0000000..0ec9a57
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicy.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+
+/**
+ * This policy can be used to trigger and evict based on a punctuation which is
+ * present within the arriving data. Using this policy, one can react on an
+ * externally defined arbitrary windowing semantic.
+ *
+ * In case this policy is used for eviction, the complete buffer will get
+ * deleted in case the punctuation is detected.
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it react on punctuation even in fake elements.
+ *
+ * @param <IN>
+ * The type of the input data handled by this policy. An
+ * {@link Extractor} can be used to extract DATA for IN.
+ * @param <DATA>
+ * The type of the punctuation. An {@link Extractor} can be used to
+ * extract DATA for IN.
+ */
+public class PunctuationPolicy<IN, DATA> implements TriggerPolicy<IN>, EvictionPolicy<IN> {
+
+ /**
+ * auto generated version id
+ */
+ private static final long serialVersionUID = -8845130188912602498L;
+ private int counter = 0;
+ private Extractor<IN, DATA> extractor;
+ private DATA punctuation;
+
+ /**
+ * Creates the punctuation policy without using any extractor. To make this
+ * work IN and DATA must not be different types.
+ *
+ * @param punctuation
+ * the punctuation which leads to trigger/evict.
+ */
+ public PunctuationPolicy(DATA punctuation) {
+ this(punctuation, null);
+ }
+
+ /**
+ * Creates the punctuation policy which uses the specified extractor to
+ * isolate the punctuation from the data.
+ *
+ * @param punctuation
+ * the punctuation which leads to trigger/evict.
+ * @param extractor
+ * An {@link Extractor} which converts IN to DATA.
+ */
+ public PunctuationPolicy(DATA punctuation, Extractor<IN, DATA> extractor) {
+ this.punctuation = punctuation;
+ this.extractor = extractor;
+ }
+
+ @Override
+ public int notifyEviction(IN datapoint, boolean triggered, int bufferSize) {
+ if (notifyTrigger(datapoint)) {
+ int tmp = counter;
+ // As the current will be add after the eviction the counter needs
+ // to be set to one already
+ counter = 1;
+ return tmp;
+ } else {
+ counter++;
+ return 0;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean notifyTrigger(IN datapoint) {
+ DATA tmp;
+
+ // eventually extract data
+ if (extractor == null) {
+ // unchecked convert (cannot check it here)
+ tmp = (DATA) datapoint;
+ } else {
+ tmp = extractor.extract(datapoint);
+ }
+
+ // compare data with punctuation
+ if (punctuation.equals(tmp)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/a6cc8afb/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
new file mode 100644
index 0000000..8224b12
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/PunctuationPolicyTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import static org.junit.Assert.*;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.extractor.FieldFromTuple;
+import org.junit.Test;
+
+public class PunctuationPolicyTest {
+
+ // This value should not effect the policy. It is changed at each call to
+ // verify this.
+ private boolean triggered = false;
+
+ @Test
+ public void PunctuationTriggerTestWithoutExtraction() {
+ PunctuationPolicy<Object, Object> policy = new PunctuationPolicy<Object, Object>(
+ new TestObject(0));
+ assertTrue("The present punctuation was not detected. (POS 1)",
+ policy.notifyTrigger(new TestObject(0)));
+ assertFalse("There was a punctuation detected which wasn't present. (POS 2)",
+ policy.notifyTrigger(new TestObject(1)));
+ }
+
+ @Test
+ public void PunctuationTriggerTestWithExtraction() {
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ PunctuationPolicy<Tuple2<Object, Object>, Object> policy = new PunctuationPolicy<Tuple2<Object, Object>, Object>(
+ new TestObject(0), new FieldFromTuple(0));
+ assertTrue("The present punctuation was not detected. (POS 3)",
+ policy.notifyTrigger(new Tuple2<Object, Object>(new TestObject(0),
+ new TestObject(1))));
+ assertFalse("There was a punctuation detected which wasn't present. (POS 4)",
+ policy.notifyTrigger(new Tuple2<Object, Object>(new TestObject(1),
+ new TestObject(0))));
+ }
+
+ @Test
+ public void PunctuationEvictionTestWithoutExtraction() {
+ // The current buffer size should not effect the test. It's therefore
+ // always 0 here.
+
+ PunctuationPolicy<Object, Object> policy = new PunctuationPolicy<Object, Object>(
+ new TestObject(0));
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 5)",
+ 0, policy.notifyEviction(new TestObject(0), (triggered = !triggered), 0));
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < i; j++) {
+ assertEquals("There was a punctuation detected which wasn't present. (POS 6)", 0,
+ policy.notifyEviction(new TestObject(1), (triggered = !triggered), 0));
+ }
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 7)",
+ i + 1, policy.notifyEviction(new TestObject(0), (triggered = !triggered), 0));
+ }
+ }
+
+ @Test
+ public void PunctuationEvictionTestWithExtraction() {
+ // The current buffer size should not effect the test. It's therefore
+ // always 0 here.
+
+ @SuppressWarnings({ "unchecked", "rawtypes" })
+ PunctuationPolicy<Tuple2<Object, Object>, Object> policy = new PunctuationPolicy<Tuple2<Object, Object>, Object>(
+ new TestObject(0), new FieldFromTuple(0));
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 10)",
+ 0, policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(0),
+ new TestObject(1)), (triggered = !triggered), 0));
+ for (int i = 0; i < 10; i++) {
+ for (int j = 0; j < i; j++) {
+ assertEquals("There was a punctuation detected which wasn't present. (POS 9)", 0,
+ policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(1),
+ new TestObject(0)), (triggered = !triggered), 0));
+ }
+ assertEquals(
+ "The present punctuation was not detected or the number of deleted tuples was wrong. (POS 10)",
+ i + 1, policy.notifyEviction(new Tuple2<Object, Object>(new TestObject(0),
+ new TestObject(1)), (triggered = !triggered), 0));
+ }
+ }
+
+ private class TestObject {
+
+ private int id;
+
+ public TestObject(int id) {
+ this.id = id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof TestObject && ((TestObject) o).getId() == this.id) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public int getId() {
+ return id;
+ }
+ }
+
+}
[27/34] incubator-flink git commit: [streaming] Introduced grouped
windowing invokable, added required cloneable policy interfaces,
and adjusted existing windowing invokable to make it usable together with the
new grouped windowing invokable.
Posted by mb...@apache.org.
[streaming] Introduced grouped windowing invokable, added required cloneable policy interfaces, and adjusted existing windowing invokable to make it usable together with the new grouped windowing invokable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/cd7bfd31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/cd7bfd31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/cd7bfd31
Branch: refs/heads/master
Commit: cd7bfd31f7b09df4933c84b4131d0b220fadcb8d
Parents: 3d11d5b
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Thu Nov 20 14:39:07 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../operator/GroupedWindowingInvokable.java | 314 +++++++++++++++++++
.../invokable/operator/WindowingInvokable.java | 134 +++++++-
.../policy/CloneableEvictionPolicy.java | 45 +++
.../policy/CloneableTriggerPolicy.java | 45 +++
4 files changed, 527 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cd7bfd31/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
new file mode 100644
index 0000000..7fc35ca
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowingInvokable.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This invokable allows windowing based on {@link TriggerPolicy} and
+ * {@link EvictionPolicy} instances including their active and cloneable
+ * versions. It is additionally aware of the creation of windows per group.
+ *
+ * A {@link KeySelector} is used to specify the key position or key extraction.
+ * The {@link ReduceFunction} will be executed on each group separately. Trigger
+ * policies might either be centralized or distributed. Eviction policies are
+ * always distributed. A distributed policy have to be a
+ * {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it will
+ * be cloned to have separated instances for each group. At the startup time the
+ * distributed policies will be stored as sample, and only clones of them will
+ * be used to maintain the groups. Therefore, each group starts with the initial
+ * policy states.
+ *
+ * While a distributed policy only gets notified with the elements belonging to
+ * the respective group, a centralized policy get notified with all arriving
+ * elements. When a centralized trigger occurred, all groups get triggered. This
+ * is done by submitting the element which caused the trigger as real element to
+ * the groups it belongs to and as fake element to all other groups. Within the
+ * groups the element might be further processed, causing more triggered,
+ * prenotifications of active distributed policies and evictions like usual.
+ *
+ * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+ * implement the
+ * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+ * method. Fake elements created on prenotification will be forwarded to all
+ * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
+ * it forwards/distributed calls all groups.
+ *
+ * @param <IN>
+ * The type of input elements handled by this operator invokable.
+ */
+public class GroupedWindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, String[]>> {
+
+ /**
+ * Auto-generated serial version UID
+ */
+ private static final long serialVersionUID = -3469545957144404137L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(GroupedWindowingInvokable.class);
+
+ private KeySelector<IN, ?> keySelector;
+ private Configuration parameters;
+ private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+ private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
+ private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
+ private Map<Object, WindowingInvokable<IN>> windowingGroups = new HashMap<Object, WindowingInvokable<IN>>();
+ private LinkedList<Thread> activePolicyThreads = new LinkedList<Thread>();
+ private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+
+ /**
+ * This constructor creates an instance of the grouped windowing invokable.
+ * A {@link KeySelector} is used to specify the key position or key
+ * extraction. The {@link ReduceFunction} will be executed on each group
+ * separately. Trigger policies might either be centralized or distributed.
+ * Eviction policies are always distributed. A distributed policy have to be
+ * a {@link CloneableTriggerPolicy} or {@link CloneableEvictionPolicy} as it
+ * will be cloned to have separated instances for each group. At the startup
+ * time the distributed policies will be stored as sample, and only clones
+ * of them will be used to maintain the groups. Therefore, each group starts
+ * with the initial policy states.
+ *
+ * While a distributed policy only gets notified with the elements belonging
+ * to the respective group, a centralized policy get notified with all
+ * arriving elements. When a centralized trigger occurred, all groups get
+ * triggered. This is done by submitting the element which caused the
+ * trigger as real element to the groups it belongs to and as fake element
+ * to all other groups. Within the groups the element might be further
+ * processed, causing more triggered, prenotifications of active distributed
+ * policies and evictions like usual.
+ *
+ * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+ * implement the
+ * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+ * method. Fake elements created on prenotification will be forwarded to all
+ * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
+ * that it forwards/distributed calls all groups.
+ *
+ * @param userFunction
+ * The user defined {@link ReduceFunction}.
+ * @param keySelector
+ * A key selector to extract the key for the groups from the
+ * input data.
+ * @param distributedTriggerPolicies
+ * Trigger policies to be distributed and maintained individually
+ * within each group.
+ * @param distributedEvictionPolicies
+ * Eviction policies to be distributed and maintained
+ * individually within each group. There are no central eviction
+ * policies because there is no central element buffer but only a
+ * buffer per group. Therefore evictions might always be done per
+ * group.
+ * @param centralTriggerPolicies
+ * Trigger policies which will only exist once at a central
+ * place. In case a central policy triggers, it will cause all
+ * groups to be emitted. (Remark: Empty groups cannot be emitted.
+ * If only one element is contained a group, this element itself
+ * is returned as aggregated result.)
+ */
+ public GroupedWindowingInvokable(ReduceFunction<IN> userFunction,
+ KeySelector<IN, ?> keySelector,
+ LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
+ LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
+ LinkedList<TriggerPolicy<IN>> centralTriggerPolicies) {
+
+ super(userFunction);
+ this.keySelector = keySelector;
+ this.centralTriggerPolicies = centralTriggerPolicies;
+ this.distributedTriggerPolicies = distributedTriggerPolicies;
+ this.distributedEvictionPolicies = distributedEvictionPolicies;
+
+ for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
+ if (trigger instanceof ActiveTriggerPolicy) {
+ this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
+ }
+ }
+ }
+
+ @Override
+ protected void immutableInvoke() throws Exception {
+ // Prevent empty data streams
+ if ((reuse = recordIterator.next(reuse)) == null) {
+ throw new RuntimeException("DataStream must not be empty");
+ }
+
+ // Continuously run
+ while (reuse != null) {
+ WindowingInvokable<IN> groupInvokable = windowingGroups.get(keySelector.getKey(reuse
+ .getObject()));
+ if (groupInvokable == null) {
+ groupInvokable = makeNewGroup(reuse);
+ }
+
+ // Run the precalls for central active triggers
+ for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
+ IN[] result = trigger.preNotifyTrigger(reuse.getObject());
+ for (IN in : result) {
+ for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ group.processFakeElement(in, trigger);
+ }
+ }
+ }
+
+ // Process non-active central triggers
+ for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
+ if (triggerPolicy.notifyTrigger(reuse.getObject())) {
+ currentTriggerPolicies.add(triggerPolicy);
+ }
+ }
+
+ if (currentTriggerPolicies.isEmpty()) {
+ // only add the element to its group
+ groupInvokable.processRealElement(reuse.getObject());
+ } else {
+ // call user function for all groups
+ for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ if (group == groupInvokable) {
+ // process real with initialized policies
+ group.processRealElement(reuse.getObject(), currentTriggerPolicies);
+ } else {
+ // process like a fake but also initialized with
+ // policies
+ group.externalTriggerFakeElement(reuse.getObject(), currentTriggerPolicies);
+ }
+ }
+ }
+
+ // clear current trigger list
+ currentTriggerPolicies.clear();
+
+ // Recreate the reuse-StremRecord object and load next StreamRecord
+ resetReuse();
+ reuse = recordIterator.next(reuse);
+ }
+
+ // Stop all remaining threads from policies
+ for (Thread t : activePolicyThreads) {
+ t.interrupt();
+ }
+
+ // finally trigger the buffer.
+ for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ group.emitFinalWindow(centralTriggerPolicies);
+ }
+
+ }
+
+ /**
+ * This method creates a new group. The method gets called in case an
+ * element arrives which has a key which was not seen before. The method
+ * created a nested {@link WindowingInvokable} and therefore created clones
+ * of all distributed trigger and eviction policies.
+ *
+ * @param element
+ * The element which leads to the generation of a new group
+ * (previously unseen key)
+ * @throws Exception
+ * In case the {@link KeySelector} throws an exception in
+ * {@link KeySelector#getKey(Object)}, the exception is not
+ * catched by this method.
+ */
+ private WindowingInvokable<IN> makeNewGroup(StreamRecord<IN> element) throws Exception {
+ // clone the policies
+ LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+ LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
+ for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
+ clonedDistributedTriggerPolicies.add(trigger.clone());
+ }
+ for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
+ clonedDistributedEvictionPolicies.add(eviction.clone());
+ }
+
+ @SuppressWarnings("unchecked")
+ WindowingInvokable<IN> groupInvokable = new WindowingInvokable<IN>(
+ (ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
+ clonedDistributedEvictionPolicies);
+ groupInvokable.initialize(collector, recordIterator, inSerializer, isMutable);
+ groupInvokable.open(this.parameters);
+ windowingGroups.put(keySelector.getKey(element.getObject()), groupInvokable);
+
+ return groupInvokable;
+ }
+
+ @Override
+ protected void mutableInvoke() throws Exception {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("There is currently no mutable implementation of this operator. Immutable version is used.");
+ }
+ immutableInvoke();
+ }
+
+ @Override
+ protected void callUserFunction() throws Exception {
+ // This method gets never called directly. The user function calls are
+ // all delegated to the invokable instanced which handle/represent the
+ // groups.
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ this.parameters = parameters;
+ for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
+ Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+ if (target != null) {
+ Thread thread = new Thread(target);
+ activePolicyThreads.add(thread);
+ thread.start();
+ }
+ }
+ };
+
+ /**
+ * This callback class allows to handle the the callbacks done by threads
+ * defined in active trigger policies
+ *
+ * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
+ */
+ private class WindowingCallback implements ActiveTriggerCallback<IN> {
+ private ActiveTriggerPolicy<IN> policy;
+
+ public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+ this.policy = policy;
+ }
+
+ @Override
+ public void sendFakeElement(IN datapoint) {
+ for (WindowingInvokable<IN> group : windowingGroups.values()) {
+ group.processFakeElement(datapoint, policy);
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cd7bfd31/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
index 52b5edf..29975f4 100644
--- a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokable.java
@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.List;
import java.util.NoSuchElementException;
public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, String[]>> {
@@ -50,6 +51,19 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
private ReduceFunction<IN> reducer;
+ /**
+ * This constructor created a windowing invokable using trigger and eviction
+ * policies.
+ *
+ * @param userFunction
+ * The user defined {@link ReduceFunction}
+ * @param triggerPolicies
+ * A list of {@link TriggerPolicy}s and/or
+ * {@link ActiveTriggerPolicy}s
+ * @param evictionPolicies
+ * A list of {@link EvictionPolicy}s and/or
+ * {@link ActiveEvictionPolicy}s
+ */
public WindowingInvokable(ReduceFunction<IN> userFunction,
LinkedList<TriggerPolicy<IN>> triggerPolicies,
LinkedList<EvictionPolicy<IN>> evictionPolicies) {
@@ -78,8 +92,8 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
super.open(parameters);
for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
- Runnable target=tp.createActiveTriggerRunnable(new WindowingCallback(tp));
- if (target!=null){
+ Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+ if (target != null) {
Thread thread = new Thread(target);
activePolicyTreads.add(thread);
thread.start();
@@ -87,6 +101,10 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
}
};
+ /**
+ * This class allows the active trigger threads to call back and push fake
+ * elements at any time.
+ */
private class WindowingCallback implements ActiveTriggerCallback<IN> {
private ActiveTriggerPolicy<IN> policy;
@@ -103,6 +121,7 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
@Override
protected void immutableInvoke() throws Exception {
+
// Prevent empty data streams
if ((reuse = recordIterator.next(reuse)) == null) {
throw new RuntimeException("DataStream must not be empty");
@@ -123,13 +142,7 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
}
// finally trigger the buffer.
- if (!buffer.isEmpty()) {
- currentTriggerPolicies.clear();
- for (TriggerPolicy<IN> policy : triggerPolicies) {
- currentTriggerPolicies.add(policy);
- }
- callUserFunctionAndLogException();
- }
+ emitFinalWindow(null);
}
@@ -142,6 +155,61 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
}
/**
+ * This method gets called in case of an grouped windowing in case central
+ * trigger occurred and the arriving element causing the trigger is not part
+ * of this group.
+ *
+ * Remark: This is NOT the same as
+ * {@link WindowingInvokable#processFakeElement(Object, TriggerPolicy)}!
+ * Here the eviction using active policies takes place after the call to the
+ * UDF. Usually it is done before when fake elements get submitted. This
+ * special behaviour is needed to allow the
+ * {@link GroupedWindowingInvokable} to send central triggers to all groups,
+ * even if the current element does not belong to the group.
+ *
+ * @param input
+ * a fake input element
+ * @param policies
+ * the list of policies which caused the call with this fake
+ * element
+ */
+ protected synchronized void externalTriggerFakeElement(IN input,
+ List<TriggerPolicy<IN>> policies) {
+
+ // Set the current triggers
+ currentTriggerPolicies.addAll(policies);
+
+ // emit
+ callUserFunctionAndLogException();
+
+ // clear the flag collection
+ currentTriggerPolicies.clear();
+
+ // Process the evictions and take care of double evictions
+ // In case there are multiple eviction policies present,
+ // only the one with the highest return value is recognized.
+ int currentMaxEviction = 0;
+ for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+ // use temporary variable to prevent multiple calls to
+ // notifyEviction
+ int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+ if (tmp > currentMaxEviction) {
+ currentMaxEviction = tmp;
+ }
+ }
+
+ for (int i = 0; i < currentMaxEviction; i++) {
+ try {
+ buffer.removeFirst();
+ } catch (NoSuchElementException e) {
+ // In case no more elements are in the buffer:
+ // Prevent failure and stop deleting.
+ break;
+ }
+ }
+ }
+
+ /**
* This method processed an arrived fake element The method is synchronized
* to ensure that it cannot interleave with
* {@link WindowingInvokable#processRealElement(Object)}
@@ -151,7 +219,8 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
* @param currentPolicy
* the policy which produced this fake element
*/
- private synchronized void processFakeElement(IN input, TriggerPolicy<IN> currentPolicy) {
+ protected synchronized void processFakeElement(IN input, TriggerPolicy<IN> currentPolicy) {
+
// Process the evictions and take care of double evictions
// In case there are multiple eviction policies present,
// only the one with the highest return value is recognized.
@@ -188,12 +257,30 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
/**
* This method processed an arrived real element The method is synchronized
* to ensure that it cannot interleave with
+ * {@link WindowingInvokable#processFakeElement(Object)}.
+ *
+ * @param input
+ * a real input element
+ * @param triggerPolicies
+ * Allows to set trigger policies which are maintained
+ * externally. This is the case for central policies in
+ * {@link GroupedWindowingInvokable}.
+ */
+ protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
+ this.currentTriggerPolicies.addAll(triggerPolicies);
+ processRealElement(input);
+ }
+
+ /**
+ * This method processed an arrived real element The method is synchronized
+ * to ensure that it cannot interleave with
* {@link WindowingInvokable#processFakeElement(Object)}
*
* @param input
* a real input element
*/
- private synchronized void processRealElement(IN input) {
+ protected synchronized void processRealElement(IN input) {
+
// Run the precalls to detect missed windows
for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
// Remark: In case multiple active triggers are present the ordering
@@ -257,6 +344,31 @@ public class WindowingInvokable<IN> extends StreamInvokable<IN, Tuple2<IN, Strin
}
+ /**
+ * This method does the final reduce at the end of the stream and emits the
+ * result.
+ *
+ * @param centralTriggerPolicies
+ * Allows to set trigger policies which are maintained
+ * externally. This is the case for central policies in
+ * {@link GroupedWindowingInvokable}.
+ */
+ protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
+ if (!buffer.isEmpty()) {
+ currentTriggerPolicies.clear();
+
+ if (centralTriggerPolicies != null) {
+ currentTriggerPolicies.addAll(centralTriggerPolicies);
+ }
+
+ for (TriggerPolicy<IN> policy : triggerPolicies) {
+ currentTriggerPolicies.add(policy);
+ }
+
+ callUserFunctionAndLogException();
+ }
+ }
+
@Override
protected void callUserFunction() throws Exception {
Iterator<IN> reducedIterator = buffer.iterator();
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cd7bfd31/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
new file mode 100644
index 0000000..0c2d691
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowingInvokable;
+
+/**
+ * When used in {@link GroupedWindowingInvokable}, eviction policies must
+ * provide a clone method. Eviction policies get cloned to provide an own
+ * instance for each group and respectively each individual element buffer as
+ * groups maintain their own buffers with the elements belonging to the
+ * respective group.
+ *
+ * This interface extends {@link EvictionPolicy} with such a clone method. It
+ * also adds the Java {@link Cloneable} interface as flag.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public interface CloneableEvictionPolicy<DATA> extends EvictionPolicy<DATA>, Cloneable {
+
+ /**
+ * This method should return an exact copy of the object it belongs to
+ * including the current object state.
+ *
+ * @return a copy of this object
+ */
+ public CloneableEvictionPolicy<DATA> clone();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/cd7bfd31/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
new file mode 100644
index 0000000..64f8bf3
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.streaming.api.invokable.operator.GroupedWindowingInvokable;
+
+/**
+ * When used in {@link GroupedWindowingInvokable}, trigger policies can provide
+ * a clone method. Cloneable triggers can can be used in a distributed manner,
+ * which means they get cloned to provide an own instance for each group. This
+ * allows each group to trigger individually and only based on the elements
+ * belonging to the respective group.
+ *
+ * This interface extends {@link TriggerPolicy} with such a clone method. It
+ * also adds the Java {@link Cloneable} interface as flag.
+ *
+ * @param <DATA>
+ * The data type handled by this policy
+ */
+public interface CloneableTriggerPolicy<DATA> extends TriggerPolicy<DATA>, Cloneable {
+
+ /**
+ * This method should return an exact copy of the object it belongs to
+ * including the current object state.
+ *
+ * @return a copy of this object
+ */
+ public CloneableTriggerPolicy<DATA> clone();
+
+}
[11/34] incubator-flink git commit: [streaming] Created trigger and
eviction policies which calculate a delta between two data points using a
given delta function. The policies evict/trigger in case the delta is higher
than a specified threshold. (incl.
Posted by mb...@apache.org.
[streaming] Created trigger and eviction policies which calculate a delta between two data points using a given delta function. The policies evict/trigger in case the delta is higher than a specified threshold. (incl. test case and the interface for delta functions)
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/997d3e1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/997d3e1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/997d3e1e
Branch: refs/heads/master
Commit: 997d3e1e9af38f2af0a6e7476ad7c868ceb404d6
Parents: fb4a839
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Mon Oct 27 13:57:26 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:08 2014 +0100
----------------------------------------------------------------------
.../windowing/deltafunction/DeltaFunction.java | 44 ++++++++
.../api/windowing/policy/DeltaPolicy.java | 107 +++++++++++++++++++
.../api/windowing/policy/DeltaPolicyTest.java | 62 +++++++++++
3 files changed, 213 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/997d3e1e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
new file mode 100644
index 0000000..b2223d6
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/deltafunction/DeltaFunction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.deltafunction;
+
+import java.io.Serializable;
+
+/**
+ * This interface allows the implementation of a function which calculates the
+ * delta between two data points. Delta functions might be used in delta
+ * policies and allow flexible adaptive windowing based on the arriving data
+ * points.
+ *
+ * @param <DATA>
+ * The type of input data which can be compared using this function.
+ */
+public interface DeltaFunction<DATA> extends Serializable {
+
+ /**
+ * Calculates the delta between two given data points.
+ *
+ * @param oldDataPoint
+ * the old data point.
+ * @param newDataPoint
+ * the new data point.
+ * @return the delta between the two given points.
+ */
+ public double getDelta(DATA oldDataPoint, DATA newDataPoint);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/997d3e1e/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
new file mode 100644
index 0000000..418f989
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicy.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+
+/**
+ * This policy calculates a delta between the data point which triggered last
+ * and the currently arrived data point. It triggers if the delta is higher than
+ * a specified threshold.
+ *
+ * In case it gets used for eviction, this policy starts from the first element
+ * of the buffer and removes all elements from the buffer which have a higher
+ * delta then the threshold. As soon as there is an element with a lower delta,
+ * the eviction stops.
+ *
+ * By default this policy does not react on fake elements. Wrap it in an
+ * {@link ActiveEvictionPolicyWrapper} to make it calculate the delta even on
+ * fake elements.
+ *
+ * @param <DATA>
+ * The type of the data points which are handled by this policy
+ */
+public class DeltaPolicy<DATA> implements TriggerPolicy<DATA>, EvictionPolicy<DATA> {
+
+ /**
+ * Auto generated version ID
+ */
+ private static final long serialVersionUID = -7797538922123394967L;
+
+ private DeltaFunction<DATA> deltaFuntion;
+ private List<DATA> windowBuffer;
+ private double threshold;
+ private DATA triggerDataPoint;
+
+ /**
+ * Crates a delta policy which calculates a delta between the data point
+ * which triggered last and the currently arrived data point. It triggers if
+ * the delta is higher than a specified threshold.
+ *
+ * In case it gets used for eviction, this policy starts from the first
+ * element of the buffer and removes all elements from the buffer which have
+ * a higher delta then the threshold. As soon as there is an element with a
+ * lower delta, the eviction stops.
+ *
+ * @param deltaFuntion
+ * The delta function to be used.
+ * @param init
+ * The initial to be used for the calculation of a delta before
+ * the first trigger.
+ * @param threshold
+ * The threshold upon which a triggering should happen.
+ */
+ public DeltaPolicy(DeltaFunction<DATA> deltaFuntion, DATA init, double threshold) {
+ this.deltaFuntion = deltaFuntion;
+ this.triggerDataPoint = init;
+ this.windowBuffer = new LinkedList<DATA>();
+ this.threshold = threshold;
+ }
+
+ @Override
+ public boolean notifyTrigger(DATA datapoint) {
+ if (deltaFuntion.getDelta(this.triggerDataPoint, datapoint) > this.threshold) {
+ this.triggerDataPoint = datapoint;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int notifyEviction(DATA datapoint, boolean triggered, int bufferSize) {
+ windowBuffer = windowBuffer.subList(windowBuffer.size() - bufferSize, bufferSize);
+ int evictCount = 0;
+ for (DATA bufferPoint : windowBuffer) {
+ if (deltaFuntion.getDelta(bufferPoint, datapoint) < this.threshold) {
+ break;
+ }
+ evictCount++;
+ }
+
+ if (evictCount > 0) {
+ windowBuffer = windowBuffer.subList(evictCount, windowBuffer.size());
+ }
+ windowBuffer.add(datapoint);
+ return evictCount;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/997d3e1e/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
new file mode 100644
index 0000000..424eb57
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/policy/DeltaPolicyTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.policy;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
+import org.apache.flink.streaming.api.windowing.policy.DeltaPolicy;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Arrays;
+
+import static org.junit.Assert.*;
+
+public class DeltaPolicyTest {
+
+ @SuppressWarnings({ "serial", "unchecked", "rawtypes" })
+ @Test
+ public void testDelta() {
+ DeltaPolicy deltaPolicy = new DeltaPolicy(new DeltaFunction<Tuple2<Integer, Integer>>() {
+ @Override
+ public double getDelta(Tuple2<Integer, Integer> oldDataPoint, Tuple2<Integer, Integer> newDataPoint) {
+ return (double) newDataPoint.f0 - oldDataPoint.f0;
+ }
+ }, new Tuple2(0, 0), 2);
+
+ List<Tuple2> tuples = Arrays.asList(
+ new Tuple2(1, 0),
+ new Tuple2(2, 0),
+ new Tuple2(3, 0),
+ new Tuple2(6, 0));
+
+ assertFalse(deltaPolicy.notifyTrigger(tuples.get(0)));
+ assertEquals(0, deltaPolicy.notifyEviction(tuples.get(0), false, 0));
+
+ assertFalse(deltaPolicy.notifyTrigger(tuples.get(1)));
+ assertEquals(0, deltaPolicy.notifyEviction(tuples.get(1), false, 1));
+
+ assertTrue(deltaPolicy.notifyTrigger(tuples.get(2)));
+ assertEquals(1, deltaPolicy.notifyEviction(tuples.get(2), true, 2));
+
+ assertTrue(deltaPolicy.notifyTrigger(tuples.get(3)));
+ assertEquals(2, deltaPolicy.notifyEviction(tuples.get(3), true, 2));
+ }
+
+
+}
\ No newline at end of file
[34/34] incubator-flink git commit: [docs] [streaming] Windowing
rework documentation update
Posted by mb...@apache.org.
[docs] [streaming] Windowing rework documentation update
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/f8440880
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/f8440880
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/f8440880
Branch: refs/heads/master
Commit: f844088051fe6c26672b42cde52e1a211e50cbbd
Parents: 783aa4a
Author: mbalassi <mb...@apache.org>
Authored: Fri Dec 5 18:22:03 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 18:22:03 2014 +0100
----------------------------------------------------------------------
docs/streaming_guide.md | 45 ++++++++++++++++++++++----------------------
1 file changed, 23 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/f8440880/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index 6038cd3..c82db77 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -268,15 +268,18 @@ With `sum`, `min`, and `max` for every incoming tuple the selected field is repl
With `minBy` and `maxBy` the output of the operator is the element with the current minimal or maximal value at the given field. If more components share the minimum or maximum value, the user can decide if the operator should return the first or last element. This can be set by the `first` boolean parameter.
+There is also an option to apply user defined aggregations with the usage of the `aggregate(…)` function of the data stream.
+
### Window operators
-Flink streaming provides very flexible windowing semantics to create arbitrary windows (also referred to as discretizations or slices) of the DataStreams and apply reduce or aggregation operations on them afterwards. Windowing can be used for instance to create rolling aggregations of the most recent N elements, where N could be defined by Time, Count or any arbitrary measure.
+Flink streaming provides very flexible windowing semantics to create arbitrary windows (also referred to as discretizations or slices) of the data streams and apply reduction or aggregation operations on the windows acquired. Windowing can be used for instance to create rolling aggregations of the most recent N elements, where N could be defined by Time, Count or any arbitrary user defined measure.
-The user can control the size (eviction) of the windows and the frequency of reduce/aggregation calls (triggers) on them in an intuitive api:
+The user can control the size (eviction) of the windows and the frequency of reduction or aggregation calls (triggers) on them in an intuitive API:
-~~~java
-dataStream.window(…).every(…).reduce/reduceGroup/aggregation
-~~~
+
+ * `dataStream.window(…).every(…).reduce(…)`
+ * `dataStream.window(…).every(…).reduceGroup(…)`
+ * `dataStream.window(…).every(…).aggregate(…)`
The next example would create windows that hold elements of the last 5 seconds, and the user defined aggregation/reduce is executed on the windows every second (sliding the window by 1 second):
@@ -284,44 +287,42 @@ The next example would create windows that hold elements of the last 5 seconds,
dataStream.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(1, TimeUnit.SECONDS))
~~~
-This approach is often referred to as policy based windowing. Different policies (count, time, etc.) can be mixed as well; for example to downsample our stream, we can create a window that will take the latest 100 elements of our stream every minute:
+This approach is often referred to as policy based windowing. Different policies (count, time, etc.) can be mixed as well; for example to downsample our stream, a window that takes the latest 100 elements of the stream every minute is created as follows:
~~~java
dataStream.window(Count.of(100)).every(Time.of(1, TimeUnit.MINUTES))
~~~
-The user can also omit the `.every(…)` call which results in a tumbling window, which empties the window after every aggregation call.
+The user can also omit the `.every(…)` call which results in a tumbling window emptying the window after every aggregation call.
-Several predefined policies are provided in the API, including delta-based, count-based and time-based policies. These can be accessed through the static methods provided by the PolicyHelper classses:
+Several predefined policies are provided in the API, including delta-based, count-based and time-based policies. These can be accessed through the static methods provided by the `PolicyHelper` classes:
-`Time.of(…)`
-`Count.of(…)`
-`Delta.of(…)`
+ * `Time.of(…)`
+ * `Count.of(…)`
+ * `Delta.of(…)`
For detailed description of these policies please refer to the javadocs.
-Now we are going to take a deeper look at the policies.
-
#### Policy based windowing
-The policy based windowing is a highly flexible way to specify your stream discretisation also called windowing semantics. Two types of policies are used for such a specification:
+The policy based windowing is a highly flexible way to specify stream discretisation also called windowing semantics. Two types of policies are used for such a specification:
-1) `TriggerPolicy` This policy defines when to trigger the reduce UDF on the current window and emit the result. In the API it completes a window statement such as: `.window(..).every(...)`, while we pass the triggering policy within `every`.
+ * `TriggerPolicy` defines when to trigger the reduce UDF on the current window and emit the result. In the API it completes a window statement such as: `.window(…).every(…)`, while the triggering policy is passed within `every`.
-When multiple triggers are used, the reduce/aggregation will be triggered at every trigger.
+When multiple triggers are used, the reduction or aggregation is executed at every trigger.
-Several predefined policies are provided in the API, including delta-based, punctuation based, count-based and time-based policies. Policies are in general UDFs and can implement any custom behaviour as well.
+Several predefined policies are provided in the API, including delta-based, punctuation based, count-based and time-based policies. Policies are in general UDFs and can implement any custom behaviour.
-2) `Eviction Policy` This policy defines the length of a window as a means of a predicate for evicting tuples when they are no longer needed. In the API this can be defined by the `.window(..)` operation on a stream. There are mostly the same predefined policy types provided as for trigger policies.
+ * `EvictionPolicy` defines the length of a window as a means of a predicate for evicting tuples when they are no longer needed. In the API this can be defined by the `.window(…)` operation on a stream. There are mostly the same predefined policy types provided as for trigger policies.
-When multiple evictions are used the strictest one will control the elements in the window. For instance in the call `ds.window(Count.of(5), Time.of(1,TimeUnit.SECONDS)).every(..)` we will get a window of max 5 elements which have arrived in the last second.
+When multiple evictions are used the strictest one controls the elements in the window. For instance in the call `dataStream.window(Count.of(5), Time.of(1,TimeUnit.SECONDS)).every(…)` produces a window of maximum 5 elements which have arrived in the last second.
+In addition to the `dataStream.window(…).every(…)` style users can specifically pass the list of trigger and eviction policies during the window call:
-In addition to the `ds.window().every()` style users can specifically pass the list of trigger and eviction policies during the window call:
~~~java
-myStream.window(ListOfTriggerPolicies,ListOfEvictionPolicies)
+dataStream.window(ListOfTriggerPolicies, ListOfEvictionPolicies)
~~~
-By default most triggers can only trigger when a new element arrives. This might not be suitable for all the use-cases, especially when time based windowing is used. You may want to trigger not only when an element arrives but also in between. Active policies provide this functionality. The predefined time-based policies are already implemented in such an active way and can hold as an example in case you want to implement your own user defined active policy.
+By default most triggers can only trigger when a new element arrives. This might not be suitable for all the use-cases, especially when time based windowing is applied. To also provide trigering between elements so called active policies can be used. The predefined time-based policies are already implemented in such a way and can hold as an example for user defined active policy implementations.
Time-based trigger and eviction policies can work with user defined `TimeStamp` implementations, these policies already cover most use cases.
[20/34] incubator-flink git commit: [streaming] Added test cases for
the policy based windowing invokable.
Posted by mb...@apache.org.
[streaming] Added test cases for the policy based windowing invokable.
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/3f506835
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/3f506835
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/3f506835
Branch: refs/heads/master
Commit: 3f506835b6d2fbe13a91e74d34de82b584cddcd1
Parents: 542c66f
Author: Jonas Traub (powibol) <jo...@s-traub.com>
Authored: Tue Oct 28 10:12:36 2014 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Fri Dec 5 16:45:09 2014 +0100
----------------------------------------------------------------------
.../operator/WindowingInvokableTest.java | 272 +++++++++++++++++++
1 file changed, 272 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/3f506835/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
new file mode 100644
index 0000000..79df1c5
--- /dev/null
+++ b/flink-addons/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowingInvokableTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.invokable.util.TimeStamp;
+import org.apache.flink.streaming.api.windowing.extractor.Extractor;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.util.MockInvokable;
+import org.junit.Test;
+
+public class WindowingInvokableTest {
+
+ /**
+ * Test case equal to {@link WindowReduceInvokableTest}
+ */
+ @Test
+ public void testWindowingInvokableWithTimePolicy() {
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ inputs.add(1);
+ inputs.add(2);
+ inputs.add(2);
+ inputs.add(3);
+ inputs.add(4);
+ inputs.add(5);
+ inputs.add(10);
+ inputs.add(11);
+ inputs.add(11);
+ // 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+ // 12-12-5-10-32
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(12);
+ expected.add(12);
+ expected.add(5);
+ expected.add(10);
+ expected.add(32);
+
+ TimeStamp<Integer> myTimeStamp = new TimeStamp<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public long getTimestamp(Integer value) {
+ return value;
+ }
+
+ @Override
+ public long getStartTime() {
+ return 1;
+ }
+ };
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger every 2 time units but delay the first trigger by 2 (First
+ // trigger after 4, then every 2)
+ triggers.add(new TimeTriggerPolicy<Integer>(2L, myTimeStamp, 2L,
+ new Extractor<Long, Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer extract(Long in) {
+ return in.intValue();
+ }
+ }));
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // Always delete all elements older then 4
+ evictions.add(new TimeEvictionPolicy<Integer>(4L, myTimeStamp));
+
+ WindowingInvokable<Integer> invokable = new WindowingInvokable<Integer>(myReduceFunction,
+ triggers, evictions);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t.f0);
+ }
+
+ assertEquals(expected, result);
+ }
+
+ /**
+ * Test case equal to {@link BatchReduceTest}
+ */
+ @Test
+ public void testWindowingInvokableWithCountPolicy() {
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ for (Integer i = 1; i <= 10; i++) {
+ inputs.add(i);
+ }
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ /*
+ * The following setup reassembles the batch size 3 and the slide size 2
+ * of the BatchReduceInvokable.
+ */
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger on every 2nd element, but the first time after the 3rd
+ triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // On every 2nd element, remove the oldest 2 elements, but the first
+ // time after the 3rd element
+ evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+ WindowingInvokable<Integer> invokable = new WindowingInvokable<Integer>(myReduceFunction,
+ triggers, evictions);
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(6);
+ expected.add(12);
+ expected.add(18);
+ expected.add(24);
+ expected.add(19);
+ List<Integer> result = new ArrayList<Integer>();
+ for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t.f0);
+ }
+ assertEquals(expected, result);
+
+ /*
+ * Begin test part 2
+ */
+
+ List<Integer> inputs2 = new ArrayList<Integer>();
+ inputs2.add(1);
+ inputs2.add(2);
+ inputs2.add(-5); // changed this value to make sure it is excluded from
+ // the result
+ inputs2.add(-3);
+ inputs2.add(-4);
+
+ myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ if (value1 <= value2) {
+ return value1;
+ } else {
+ return value2;
+ }
+ };
+ };
+
+ /*
+ * The following setup reassembles the batch size 2 and the slide size 3
+ * of the BatchReduceInvokable.
+ */
+ triggers = new LinkedList<TriggerPolicy<Integer>>();
+ // Trigger after every 3rd element, but the first time after the 2nd
+ triggers.add(new CountTriggerPolicy<Integer>(3, 1));
+
+ evictions = new LinkedList<EvictionPolicy<Integer>>();
+ // On every 3rd element, remove the oldest 3 elements, but the first
+ // time after on the 5th element
+ evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
+
+ WindowingInvokable<Integer> invokable2 = new WindowingInvokable<Integer>(myReduceFunction,
+ triggers, evictions);
+
+ List<Integer> expected2 = new ArrayList<Integer>();
+ expected2.add(1);
+ expected2.add(-4);
+
+ result = new ArrayList<Integer>();
+ for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable2, inputs2)) {
+ result.add(t.f0);
+ }
+
+ assertEquals(expected2, result);
+
+ }
+
+ @Test
+ public void testWindowingInvokableWithMultiplePolicies() {
+ LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+ triggers.add(new CountTriggerPolicy<Integer>(2));
+ triggers.add(new CountTriggerPolicy<Integer>(3));
+
+ LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+ evictions.add(new CountEvictionPolicy<Integer>(2,2));
+ evictions.add(new CountEvictionPolicy<Integer>(3,3));
+
+ List<Integer> inputs = new ArrayList<Integer>();
+ for (Integer i = 1; i <= 10; i++) {
+ inputs.add(i);
+ }
+ /**
+ * <code>
+ * VAL: 1,2,3,4,5,6,7,8,9,10
+ * TR1: | | | | |
+ * TR2: | | |
+ * EV1: 2 2 2 2 2
+ * EV2: 3 3 3
+ * </code>
+ */
+
+ List<Integer> expected = new ArrayList<Integer>();
+ expected.add(3);
+ expected.add(3);
+ expected.add(4);
+ expected.add(11);
+ expected.add(15);
+ expected.add(9);
+ expected.add(10);
+
+ ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Integer reduce(Integer value1, Integer value2) throws Exception {
+ return value1 + value2;
+ }
+ };
+
+ WindowingInvokable<Integer> invokable = new WindowingInvokable<Integer>(myReduceFunction,
+ triggers, evictions);
+
+ ArrayList<Integer> result = new ArrayList<Integer>();
+ for (Tuple2<Integer, String[]> t : MockInvokable.createAndExecute(invokable, inputs)) {
+ result.add(t.f0);
+ }
+
+ assertEquals(expected, result);
+ }
+
+}