You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2015/02/16 15:25:27 UTC

[01/20] flink git commit: [streaming] StreamDiscretizer rework to support only 1 eviction and trigger for robustness + test cleanup

Repository: flink
Updated Branches:
  refs/heads/master 055f6dc3e -> e30a2ea2f


http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
new file mode 100644
index 0000000..8fcac8c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class StreamDiscretizerTest {
+
+	/**
+	 * Test case equal to {@link WindowReduceInvokableTest}
+	 */
+	@Test
+	public void testWindowInvokableWithTimePolicy() {
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(2);
+		inputs.add(2);
+		inputs.add(3);
+		inputs.add(4);
+		inputs.add(5);
+		inputs.add(10);
+		inputs.add(11);
+		inputs.add(11);
+
+		List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
+		expected.add(StreamWindow.fromElements(1, 2, 2, 3, 4));
+		expected.add(StreamWindow.fromElements(3, 4, 5));
+		expected.add(StreamWindow.fromElements(5));
+		expected.add(StreamWindow.fromElements(10));
+		expected.add(StreamWindow.fromElements(10, 11, 11));
+
+		Timestamp<Integer> myTimeStamp = new Timestamp<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public long getTimestamp(Integer value) {
+				return value;
+			}
+		};
+
+		TriggerPolicy<Integer> trigger = new TimeTriggerPolicy<Integer>(2L,
+				new TimestampWrapper<Integer>(myTimeStamp, 1), 2L);
+
+		EvictionPolicy<Integer> eviction = new TimeEvictionPolicy<Integer>(4L,
+				new TimestampWrapper<Integer>(myTimeStamp, 1));
+
+		StreamDiscretizer<Integer> discretizer = new StreamDiscretizer<Integer>(trigger, eviction);
+
+		List<StreamWindow<Integer>> result = MockContext.createAndExecute(discretizer, inputs);
+		assertEquals(expected, result);
+	}
+
+	@Test
+	public void testWindowInvokableWithCountPolicy() {
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(2);
+		inputs.add(2);
+		inputs.add(3);
+		inputs.add(4);
+
+		List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
+		expected.add(StreamWindow.fromElements(1, 2));
+		expected.add(StreamWindow.fromElements(2, 3));
+		expected.add(StreamWindow.fromElements(4));
+
+		TriggerPolicy<Integer> trigger = new CountTriggerPolicy<Integer>(2);
+
+		EvictionPolicy<Integer> eviction = new TumblingEvictionPolicy<Integer>();
+
+		StreamDiscretizer<Integer> discretizer = new StreamDiscretizer<Integer>(trigger, eviction);
+
+		List<StreamWindow<Integer>> result = MockContext.createAndExecute(discretizer, inputs);
+		assertEquals(expected, result);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
index 66a11f5..5ceecb1 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
@@ -100,16 +100,26 @@ public class StreamWindowTest {
 		assertEquals(window1.windowID, merged2.windowID);
 		assertEquals(values, new HashSet<Integer>(merged2));
 
+	}
+
+	@Test
+	public void serializerTest() throws IOException {
+
+		StreamWindow<Integer> streamWindow = new StreamWindow<Integer>();
+		streamWindow.add(1);
+		streamWindow.add(2);
+		streamWindow.add(3);
+
 		TypeSerializer<StreamWindow<Integer>> ts = new StreamWindowSerializer<Integer>(
 				BasicTypeInfo.INT_TYPE_INFO);
 
 		TestOutputView ow = new TestOutputView();
 
-		ts.serialize(merged2, ow);
+		ts.serialize(streamWindow, ow);
 
 		TestInputView iw = ow.getInputView();
 
-		System.out.println(ts.deserialize(iw));
+		assertEquals(streamWindow, ts.deserialize(iw));
 
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java
deleted file mode 100644
index bba134a..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class WindowInvokableTest {
-
-	/**
-	 * Test case equal to {@link WindowReduceInvokableTest}
-	 */
-	@Test
-	public void testWindowInvokableWithTimePolicy() {
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(2);
-		inputs.add(2);
-		inputs.add(3);
-		inputs.add(4);
-		inputs.add(5);
-		inputs.add(10);
-		inputs.add(11);
-		inputs.add(11);
-		// 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
-		// 12-12-5-10-32
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(12);
-		expected.add(12);
-		expected.add(5);
-		expected.add(10);
-		expected.add(32);
-
-		Timestamp<Integer> myTimeStamp = new Timestamp<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public long getTimestamp(Integer value) {
-				return value;
-			}
-		};
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		// Trigger every 2 time units but delay the first trigger by 2 (First
-		// trigger after 4, then every 2)
-		triggers.add(new TimeTriggerPolicy<Integer>(2L, new TimestampWrapper<Integer>(myTimeStamp,
-				1), 2L));
-
-		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
-		// Always delete all elements older then 4
-		evictions.add(new TimeEvictionPolicy<Integer>(4L, new TimestampWrapper<Integer>(
-				myTimeStamp, 1)));
-
-		WindowInvokable<Integer, Integer> invokable = new WindowReducer<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(expected, result);
-	}
-
-	/**
-	 * Test case equal to {@link BatchReduceTest}
-	 */
-	@Test
-	public void testWindowInvokableWithCountPolicy() {
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		for (Integer i = 1; i <= 10; i++) {
-			inputs.add(i);
-		}
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		/*
-		 * The following setup reassembles the batch size 3 and the slide size 2
-		 * of the BatchReduceInvokable.
-		 */
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		// Trigger on every 2nd element, but the first time after the 3rd
-		triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
-		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
-		// On every 2nd element, remove the oldest 2 elements, but the first
-		// time after the 3rd element
-		evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
-		WindowInvokable<Integer, Integer> invokable = new WindowReducer<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(6);
-		expected.add(12);
-		expected.add(18);
-		expected.add(24);
-		expected.add(19);
-		List<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-		assertEquals(expected, result);
-
-		/*
-		 * Begin test part 2
-		 */
-
-		List<Integer> inputs2 = new ArrayList<Integer>();
-		inputs2.add(1);
-		inputs2.add(2);
-		inputs2.add(-5); // changed this value to make sure it is excluded from
-							// the result
-		inputs2.add(-3);
-		inputs2.add(-4);
-
-		myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				if (value1 <= value2) {
-					return value1;
-				} else {
-					return value2;
-				}
-			};
-		};
-
-		/*
-		 * The following setup reassembles the batch size 2 and the slide size 3
-		 * of the BatchReduceInvokable.
-		 */
-		triggers = new LinkedList<TriggerPolicy<Integer>>();
-		// Trigger after every 3rd element, but the first time after the 2nd
-		triggers.add(new CountTriggerPolicy<Integer>(3, 1));
-
-		evictions = new LinkedList<EvictionPolicy<Integer>>();
-		// On every 3rd element, remove the oldest 3 elements, but the first
-		// time after on the 5th element
-		evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
-
-		WindowInvokable<Integer, Integer> invokable2 = new WindowReducer<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		List<Integer> expected2 = new ArrayList<Integer>();
-		expected2.add(1);
-		expected2.add(-4);
-
-		result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable2, inputs2)) {
-			result.add(t);
-		}
-
-		assertEquals(expected2, result);
-
-	}
-
-	@Test
-	public void testWindowInvokableWithMultiplePolicies() {
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		triggers.add(new CountTriggerPolicy<Integer>(2));
-		triggers.add(new CountTriggerPolicy<Integer>(3));
-
-		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
-		evictions.add(new CountEvictionPolicy<Integer>(2, 2));
-		evictions.add(new CountEvictionPolicy<Integer>(3, 3));
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		for (Integer i = 1; i <= 10; i++) {
-			inputs.add(i);
-		}
-		/**
-		 * <code>
-		 * VAL: 1,2,3,4,5,6,7,8,9,10
-		 * TR1:   |   |   |   |   |
-		 * TR2:     |     |     |
-		 * EV1:   2   2   2   2   2
-		 * EV2:     3     3     3
-		 * </code>
-		 */
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(3);
-		expected.add(3);
-		expected.add(4);
-		expected.add(11);
-		expected.add(15);
-		expected.add(9);
-		expected.add(10);
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		WindowInvokable<Integer, Integer> invokable = new WindowReducer<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(expected, result);
-	}
-
-}


[15/20] flink git commit: [streaming] WindowMapFunction added + Streaming package structure cleanup

Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
index ed6b347..a430700 100755
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
@@ -24,11 +24,12 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 
-import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.streaming.api.windowing.helper.Count;
 import org.apache.flink.streaming.util.TestStreamEnvironment;
 import org.apache.flink.util.Collector;
@@ -55,10 +56,10 @@ public class WindowIntegrationTest implements Serializable {
 
 	@SuppressWarnings("serial")
 	public static class IdentityWindowMap implements
-			GroupReduceFunction<Integer, StreamWindow<Integer>> {
+			WindowMapFunction<Integer, StreamWindow<Integer>> {
 
 		@Override
-		public void reduce(Iterable<Integer> values, Collector<StreamWindow<Integer>> out)
+		public void mapWindow(Iterable<Integer> values, Collector<StreamWindow<Integer>> out)
 				throws Exception {
 
 			StreamWindow<Integer> window = new StreamWindow<Integer>();

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/StreamWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/StreamWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/StreamWindowTest.java
new file mode 100644
index 0000000..a07b681
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/StreamWindowTest.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.api.windowing.StreamWindowSerializer;
+import org.junit.Test;
+
+public class StreamWindowTest {
+
+	@Test
+	public void creationTest() {
+
+		StreamWindow<Integer> window1 = new StreamWindow<Integer>();
+		assertTrue(window1.isEmpty());
+		assertTrue(window1.windowID != 0);
+
+		window1.add(10);
+		assertEquals(1, window1.size());
+
+		StreamWindow<Integer> window2 = new StreamWindow<Integer>(window1);
+
+		assertTrue(window1.windowID == window2.windowID);
+		assertEquals(1, window2.size());
+
+		StreamWindow<Integer> window3 = new StreamWindow<Integer>(100);
+		assertEquals(100, window3.windowID);
+
+		StreamWindow<Integer> window4 = new StreamWindow<Integer>();
+		assertFalse(window4.windowID == window1.windowID);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void mergeTest() throws IOException {
+		StreamWindow<Integer> window1 = new StreamWindow<Integer>().setNumberOfParts(3);
+		StreamWindow<Integer> window2 = new StreamWindow<Integer>(window1.windowID, 3);
+		StreamWindow<Integer> window3 = new StreamWindow<Integer>(window1.windowID, 3);
+
+		window1.add(1);
+		window2.add(2);
+		window3.add(3);
+
+		Set<Integer> values = new HashSet<Integer>();
+		values.add(1);
+		values.add(2);
+		values.add(3);
+
+		StreamWindow<Integer> merged = StreamWindow.merge(window1, window2, window3);
+
+		assertEquals(3, merged.size());
+		assertEquals(window1.windowID, merged.windowID);
+		assertEquals(values, new HashSet<Integer>(merged));
+
+		try {
+			StreamWindow.merge(window1, new StreamWindow<Integer>());
+			fail();
+		} catch (RuntimeException e) {
+			// good
+		}
+
+		List<StreamWindow<Integer>> wList = merged.split(3);
+
+		StreamWindow<Integer> merged2 = StreamWindow.merge(wList);
+
+		assertEquals(3, merged2.size());
+		assertEquals(window1.windowID, merged2.windowID);
+		assertEquals(values, new HashSet<Integer>(merged2));
+
+	}
+
+	@Test
+	public void serializerTest() throws IOException {
+
+		StreamWindow<Integer> streamWindow = new StreamWindow<Integer>();
+		streamWindow.add(1);
+		streamWindow.add(2);
+		streamWindow.add(3);
+
+		TypeSerializer<StreamWindow<Integer>> ts = new StreamWindowSerializer<Integer>(
+				BasicTypeInfo.INT_TYPE_INFO, null);
+
+		TestOutputView ow = new TestOutputView();
+
+		ts.serialize(streamWindow, ow);
+
+		TestInputView iw = ow.getInputView();
+
+		assertEquals(streamWindow, ts.deserialize(iw));
+
+	}
+
+	@Test
+	public void partitionTest() {
+		StreamWindow<Integer> streamWindow = new StreamWindow<Integer>();
+		streamWindow.add(1);
+		streamWindow.add(2);
+		streamWindow.add(3);
+		streamWindow.add(4);
+		streamWindow.add(5);
+		streamWindow.add(6);
+
+		List<StreamWindow<Integer>> split = streamWindow.split(2);
+		assertEquals(2, split.size());
+		assertEquals(StreamWindow.fromElements(1, 2, 3), split.get(0));
+		assertEquals(StreamWindow.fromElements(4, 5, 6), split.get(1));
+
+		List<StreamWindow<Integer>> split2 = streamWindow.split(6);
+		assertEquals(6, split2.size());
+		assertEquals(StreamWindow.fromElements(1), split2.get(0));
+		assertEquals(StreamWindow.fromElements(2), split2.get(1));
+		assertEquals(StreamWindow.fromElements(3), split2.get(2));
+		assertEquals(StreamWindow.fromElements(4), split2.get(3));
+		assertEquals(StreamWindow.fromElements(5), split2.get(4));
+		assertEquals(StreamWindow.fromElements(6), split2.get(5));
+
+		List<StreamWindow<Integer>> split3 = streamWindow.split(10);
+		assertEquals(6, split3.size());
+		assertEquals(StreamWindow.fromElements(1), split3.get(0));
+		assertEquals(StreamWindow.fromElements(2), split3.get(1));
+		assertEquals(StreamWindow.fromElements(3), split3.get(2));
+		assertEquals(StreamWindow.fromElements(4), split3.get(3));
+		assertEquals(StreamWindow.fromElements(5), split3.get(4));
+		assertEquals(StreamWindow.fromElements(6), split3.get(5));
+
+	}
+
+	private class TestOutputView extends DataOutputStream implements DataOutputView {
+
+		public TestOutputView() {
+			super(new ByteArrayOutputStream(4096));
+		}
+
+		public TestInputView getInputView() {
+			ByteArrayOutputStream baos = (ByteArrayOutputStream) out;
+			return new TestInputView(baos.toByteArray());
+		}
+
+		@Override
+		public void skipBytesToWrite(int numBytes) throws IOException {
+			for (int i = 0; i < numBytes; i++) {
+				write(0);
+			}
+		}
+
+		@Override
+		public void write(DataInputView source, int numBytes) throws IOException {
+			byte[] buffer = new byte[numBytes];
+			source.readFully(buffer);
+			write(buffer);
+		}
+	}
+
+	private class TestInputView extends DataInputStream implements DataInputView {
+
+		public TestInputView(byte[] data) {
+			super(new ByteArrayInputStream(data));
+		}
+
+		@Override
+		public void skipBytesToRead(int numBytes) throws IOException {
+			while (numBytes > 0) {
+				int skipped = skipBytes(numBytes);
+				numBytes -= skipped;
+			}
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBufferTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBufferTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBufferTest.java
new file mode 100644
index 0000000..bb756c7
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBufferTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.api.windowing.windowbuffer.BasicWindowBuffer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class BasicWindowBufferTest {
+
+	@Test
+	public void testEmitWindow() throws Exception {
+
+		TestCollector<StreamWindow<Integer>> collector = new TestCollector<StreamWindow<Integer>>();
+		List<StreamWindow<Integer>> collected = collector.getCollected();
+
+		WindowBuffer<Integer> wb = new BasicWindowBuffer<Integer>();
+
+		wb.store(2);
+		wb.store(10);
+
+		assertEquals(2, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(1, collected.size());
+		assertEquals(StreamWindow.fromElements(2, 10), collected.get(0));
+
+		wb.store(4);
+		wb.evict(2);
+
+		assertEquals(1, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(2, collected.size());
+		assertEquals(StreamWindow.fromElements(4), collected.get(1));
+
+		wb.evict(1);
+
+		wb.emitWindow(collector);
+		assertEquals(2, collected.size());
+	}
+
+	public static class TestCollector<T> implements Collector<T> {
+
+		private final List<T> collected = new ArrayList<T>();
+
+		@Override
+		public void collect(T record) {
+			collected.add(record);
+		}
+
+		@Override
+		public void close() {
+		}
+
+		public List<T> getCollected() {
+			return collected;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducerTest.java
new file mode 100644
index 0000000..437fcd6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducerTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.api.windowing.windowbuffer.TumblingGroupedPreReducer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.BasicWindowBufferTest.TestCollector;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
+import org.junit.Test;
+
+public class TumblingGroupedPreReducerTest {
+
+	TypeInformation<Tuple2<Integer, Integer>> type = TypeExtractor
+			.getForObject(new Tuple2<Integer, Integer>(1, 1));
+	TypeSerializer<Tuple2<Integer, Integer>> serializer = type.createSerializer(null);
+
+	KeySelector<Tuple2<Integer, Integer>, ?> key = KeySelectorUtil.getSelectorForKeys(
+			new Keys.ExpressionKeys<Tuple2<Integer, Integer>>(new int[] { 0 }, type), type, null);
+
+	Reducer reducer = new Reducer();
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testEmitWindow() throws Exception {
+
+		List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
+		inputs.add(new Tuple2<Integer, Integer>(1, 1));
+		inputs.add(new Tuple2<Integer, Integer>(0, 0));
+		inputs.add(new Tuple2<Integer, Integer>(1, -1));
+		inputs.add(new Tuple2<Integer, Integer>(1, -2));
+
+		TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector = new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
+		List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
+
+		WindowBuffer<Tuple2<Integer, Integer>> wb = new TumblingGroupedPreReducer<Tuple2<Integer, Integer>>(
+				reducer, key, serializer);
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+
+		assertEquals(2, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(1, collected.size());
+
+		assertSetEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(1, 1),
+				new Tuple2<Integer, Integer>(0, 0)), collected.get(0));
+
+		// Test automatic eviction
+		assertEquals(0, wb.size());
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+		wb.store(serializer.copy(inputs.get(2)));
+
+		// Nothing should happen here
+		wb.evict(3);
+
+		wb.store(serializer.copy(inputs.get(3)));
+
+		assertEquals(4, wb.size());
+		wb.emitWindow(collector);
+
+		assertEquals(2, collected.size());
+
+		assertSetEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(3, -2),
+				new Tuple2<Integer, Integer>(0, 0)), collected.get(1));
+
+		// Test whether function is mutating inputs or not
+		assertEquals(2, reducer.allInputs.size());
+		assertEquals(reducer.allInputs.get(0), inputs.get(2));
+		assertEquals(reducer.allInputs.get(1), inputs.get(3));
+
+	}
+
+	private static <T> void assertSetEquals(Collection<T> first, Collection<T> second) {
+		assertEquals(new HashSet<T>(first), new HashSet<T>(second));
+	}
+
+	@SuppressWarnings("serial")
+	private class Reducer implements ReduceFunction<Tuple2<Integer, Integer>> {
+
+		public List<Tuple2<Integer, Integer>> allInputs = new ArrayList<Tuple2<Integer, Integer>>();
+
+		@Override
+		public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1,
+				Tuple2<Integer, Integer> value2) throws Exception {
+			allInputs.add(value2);
+			value1.f0 = value1.f0 + value2.f0;
+			value1.f1 = value1.f1 + value2.f1;
+			return value1;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducerTest.java
new file mode 100644
index 0000000..31be227
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducerTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.api.windowing.windowbuffer.TumblingPreReducer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.BasicWindowBufferTest.TestCollector;
+import org.junit.Test;
+
+public class TumblingPreReducerTest {
+
+	TypeSerializer<Tuple2<Integer, Integer>> serializer = TypeExtractor.getForObject(
+			new Tuple2<Integer, Integer>(1, 1)).createSerializer(null);
+
+	Reducer reducer = new Reducer();
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testEmitWindow() throws Exception {
+
+		List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
+		inputs.add(new Tuple2<Integer, Integer>(1, 1));
+		inputs.add(new Tuple2<Integer, Integer>(2, 0));
+		inputs.add(new Tuple2<Integer, Integer>(3, -1));
+		inputs.add(new Tuple2<Integer, Integer>(4, -2));
+
+		TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector = new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
+		List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
+
+		WindowBuffer<Tuple2<Integer, Integer>> wb = new TumblingPreReducer<Tuple2<Integer, Integer>>(
+				reducer, serializer);
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+
+		assertEquals(2, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(1, collected.size());
+		assertEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(3, 1)),
+				collected.get(0));
+
+		// Test automatic eviction
+		assertEquals(0, wb.size());
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+		wb.store(serializer.copy(inputs.get(2)));
+
+		// Nothing should happen here
+		wb.evict(3);
+
+		wb.store(serializer.copy(inputs.get(3)));
+
+		assertEquals(4, wb.size());
+		wb.emitWindow(collector);
+
+		assertEquals(2, collected.size());
+		assertEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(10, -2)),
+				collected.get(1));
+
+		// Test whether function is mutating inputs or not
+		assertEquals(4, reducer.allInputs.size());
+		assertEquals(reducer.allInputs.get(0), inputs.get(1));
+		assertEquals(reducer.allInputs.get(1), inputs.get(1));
+		assertEquals(reducer.allInputs.get(2), inputs.get(2));
+		assertEquals(reducer.allInputs.get(3), inputs.get(3));
+
+	}
+
+	@SuppressWarnings("serial")
+	private class Reducer implements ReduceFunction<Tuple2<Integer, Integer>> {
+
+		public List<Tuple2<Integer, Integer>> allInputs = new ArrayList<Tuple2<Integer, Integer>>();
+
+		@Override
+		public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1,
+				Tuple2<Integer, Integer> value2) throws Exception {
+			allInputs.add(value2);
+			value1.f0 = value1.f0 + value2.f0;
+			value1.f1 = value1.f1 + value2.f1;
+			return value1;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
index a61adac..4cdb7c6 100755
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
@@ -19,9 +19,9 @@ package org.apache.flink.streaming.examples.ml;
 
 import java.util.concurrent.TimeUnit;
 
-import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.function.co.CoMapFunction;
 import org.apache.flink.streaming.api.function.source.SourceFunction;
 import org.apache.flink.streaming.api.windowing.helper.Time;
@@ -131,7 +131,7 @@ public class IncrementalLearningSkeleton {
 	/**
 	 * Builds up-to-date partial models on new training data.
 	 */
-	public static class PartialModelBuilder implements GroupReduceFunction<Integer, Double[]> {
+	public static class PartialModelBuilder implements WindowMapFunction<Integer, Double[]> {
 		private static final long serialVersionUID = 1L;
 
 		protected Double[] buildPartialModel(Iterable<Integer> values) {
@@ -139,7 +139,7 @@ public class IncrementalLearningSkeleton {
 		}
 
 		@Override
-		public void reduce(Iterable<Integer> values, Collector<Double[]> out) throws Exception {
+		public void mapWindow(Iterable<Integer> values, Collector<Double[]> out) throws Exception {
 			out.collect(buildPartialModel(values));
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
index af10e52..9bf851e 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
@@ -17,27 +17,27 @@
 
 package org.apache.flink.streaming.examples.windowing;
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.flink.api.common.functions.FilterFunction;
 import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.JoinFunction;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.WindowedDataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.function.source.SourceFunction;
 import org.apache.flink.streaming.api.windowing.deltafunction.DeltaFunction;
 import org.apache.flink.streaming.api.windowing.helper.Delta;
 import org.apache.flink.streaming.api.windowing.helper.Time;
 import org.apache.flink.util.Collector;
 
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
 /**
  * This example showcases a moderately complex Flink Streaming pipeline.
  * It to computes statistics on stock market data that arrive continuously,
@@ -120,7 +120,7 @@ public class StockPrices {
 
 		DataStream<StockPrice> lowest = windowedStream.minBy("price").flatten();
 		DataStream<StockPrice> maxByStock = windowedStream.groupBy("symbol").maxBy("price").flatten();
-		DataStream<StockPrice> rollingMean = windowedStream.groupBy("symbol").mapWindow(new MeanReduce()).flatten();
+		DataStream<StockPrice> rollingMean = windowedStream.groupBy("symbol").mapWindow(new WindowMean()).flatten();
 
 		//Step 3
 		//Use  delta policy to create price change warnings, and also count the number of warning every half minute
@@ -188,7 +188,7 @@ public class StockPrices {
 
 		DataStream<Double> rollingCorrelation = tweetsAndWarning
 				.window(Time.of(30, TimeUnit.SECONDS))
-				.mapWindow(new CorrelationReduce()).flatten();
+				.mapWindow(new WindowCorrelation()).flatten();
 
 		if (fileOutput) {
 			rollingCorrelation.writeAsText(outputPath, 1);
@@ -279,7 +279,7 @@ public class StockPrices {
 		}
 	}
 
-	public final static class MeanReduce implements GroupReduceFunction<StockPrice, StockPrice> {
+	public final static class WindowMean implements WindowMapFunction<StockPrice, StockPrice> {
 
 		private static final long serialVersionUID = 1L;
 		private Double sum = 0.0;
@@ -287,7 +287,7 @@ public class StockPrices {
 		private String symbol = "";
 
 		@Override
-		public void reduce(Iterable<StockPrice> values, Collector<StockPrice> out) throws Exception {
+		public void mapWindow(Iterable<StockPrice> values, Collector<StockPrice> out) throws Exception {
 			if (values.iterator().hasNext()) {
 
 				for (StockPrice sp : values) {
@@ -324,19 +324,19 @@ public class StockPrices {
 		}
 	}
 
-	public static final class SendWarning implements GroupReduceFunction<StockPrice, String> {
+	public static final class SendWarning implements WindowMapFunction<StockPrice, String> {
 
 		private static final long serialVersionUID = 1L;
 
 		@Override
-		public void reduce(Iterable<StockPrice> values, Collector<String> out) throws Exception {
+		public void mapWindow(Iterable<StockPrice> values, Collector<String> out) throws Exception {
 			if (values.iterator().hasNext()) {
 				out.collect(values.iterator().next().symbol);
 			}
 		}
 	}
 
-	public static final class CorrelationReduce implements GroupReduceFunction<Tuple2<Integer, Integer>, Double> {
+	public static final class WindowCorrelation implements WindowMapFunction<Tuple2<Integer, Integer>, Double> {
 
 		private static final long serialVersionUID = 1L;
 		private Integer leftSum;
@@ -351,7 +351,7 @@ public class StockPrices {
 		private Double rightSd;
 
 		@Override
-		public void reduce(Iterable<Tuple2<Integer, Integer>> values, Collector<Double> out) throws Exception {
+		public void mapWindow(Iterable<Tuple2<Integer, Integer>> values, Collector<Double> out) throws Exception {
 
 			leftSum = 0;
 			rightSum = 0;

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
index 780d703..33d71be 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -18,24 +18,22 @@
 
 package org.apache.flink.streaming.api.scala
 
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
-
 import scala.Array.canBuildFrom
 import scala.collection.JavaConversions.iterableAsScalaIterable
 import scala.reflect.ClassTag
 
-import org.apache.flink.api.common.functions.GroupReduceFunction
 import org.apache.flink.api.common.functions.ReduceFunction
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.functions.KeySelector
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase
 import org.apache.flink.api.streaming.scala.ScalaStreamingAggregator
 import org.apache.flink.streaming.api.datastream.{WindowedDataStream => JavaWStream}
+import org.apache.flink.streaming.api.function.WindowMapFunction
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType
 import org.apache.flink.streaming.api.function.aggregation.SumFunction
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
+import org.apache.flink.streaming.api.windowing.StreamWindow
 import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
-import org.apache.flink.streaming.api.windowing.helper._
-import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow
 import org.apache.flink.util.Collector
 
 class WindowedDataStream[T](javaStream: JavaWStream[T]) {
@@ -144,7 +142,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
    * </br>
    * Whenever possible try to use reduce instead of groupReduce for increased efficiency
    */
-  def mapWindow[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
+  def mapWindow[R: ClassTag: TypeInformation](reducer: WindowMapFunction[T, R]):
   WindowedDataStream[R] = {
     if (reducer == null) {
       throw new NullPointerException("GroupReduce function must not be null.")
@@ -165,9 +163,9 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
     if (fun == null) {
       throw new NullPointerException("GroupReduce function must not be null.")
     }
-    val reducer = new GroupReduceFunction[T, R] {
+    val reducer = new WindowMapFunction[T, R] {
       val cleanFun = clean(fun)
-      def reduce(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
+      def mapWindow(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
     }
     mapWindow(reducer)
   }

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
index cdda8f2..ae7b520 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
@@ -27,7 +27,7 @@ import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWSt
 import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
 import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
 import language.implicitConversions
-import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow
+import org.apache.flink.streaming.api.windowing.StreamWindow
 
 package object scala {
   // We have this here so that we always have generated TypeInformationS when


[19/20] flink git commit: [streaming] Integration test added for windowed operations

Posted by mb...@apache.org.
[streaming] Integration test added for windowed operations


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f32990ff
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f32990ff
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f32990ff

Branch: refs/heads/master
Commit: f32990ffa08fbe0b004c3a7443b295335a2aca3d
Parents: 412779f
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Feb 8 13:10:00 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../windowing/WindowIntegrationTest.java        | 200 +++++++++++++++++++
 1 file changed, 200 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/f32990ff/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
new file mode 100755
index 0000000..c1687e6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.streaming.api.windowing.helper.Count;
+import org.apache.flink.streaming.util.TestStreamEnvironment;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class WindowIntegrationTest implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+	private static final Integer MEMORYSIZE = 32;
+
+	@SuppressWarnings("serial")
+	private static class ModKey implements KeySelector<Integer, Integer> {
+		private int m;
+
+		public ModKey(int m) {
+			this.m = m;
+		}
+
+		@Override
+		public Integer getKey(Integer value) throws Exception {
+			return value % m;
+		}
+	}
+
+	@SuppressWarnings("serial")
+	private static class IdentityWindowMap implements
+			GroupReduceFunction<Integer, StreamWindow<Integer>> {
+
+		@Override
+		public void reduce(Iterable<Integer> values, Collector<StreamWindow<Integer>> out)
+				throws Exception {
+
+			StreamWindow<Integer> window = new StreamWindow<Integer>();
+
+			for (Integer value : values) {
+				window.add(value);
+			}
+			out.collect(window);
+		}
+
+	}
+
+	@Test
+	public void test() throws Exception {
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(2);
+		inputs.add(2);
+		inputs.add(3);
+		inputs.add(4);
+		inputs.add(5);
+		inputs.add(10);
+		inputs.add(11);
+		inputs.add(11);
+
+		StreamExecutionEnvironment env = new TestStreamEnvironment(2, MEMORYSIZE);
+
+		DataStream<Integer> source = env.fromCollection(inputs);
+
+		source.window(Count.of(2)).every(Count.of(3)).sum(0).getDiscretizedStream()
+				.addSink(new CentralSink1());
+
+		source.window(Count.of(4)).groupBy(new ModKey(2)).mapWindow(new IdentityWindowMap())
+				.flatten().addSink(new CentralSink2());
+
+		source.groupBy(new ModKey(3)).window(Count.of(2)).sum(0).getDiscretizedStream()
+				.addSink(new DistributedSink1());
+
+		source.groupBy(new ModKey(3)).window(Count.of(2)).groupBy(new ModKey(2))
+				.mapWindow(new IdentityWindowMap()).flatten().addSink(new DistributedSink2());
+
+		env.execute();
+
+		// sum ( Count of 2 slide 3 )
+		List<StreamWindow<Integer>> expected1 = new ArrayList<StreamWindow<Integer>>();
+		expected1.add(StreamWindow.fromElements(4));
+		expected1.add(StreamWindow.fromElements(9));
+		expected1.add(StreamWindow.fromElements(22));
+
+		validateOutput(expected1, CentralSink1.windows);
+
+		// Tumbling Count of 4 grouped by mod 2
+		List<StreamWindow<Integer>> expected2 = new ArrayList<StreamWindow<Integer>>();
+		expected2.add(StreamWindow.fromElements(2, 2));
+		expected2.add(StreamWindow.fromElements(1, 3));
+		expected2.add(StreamWindow.fromElements(4, 10));
+		expected2.add(StreamWindow.fromElements(5, 11));
+		expected2.add(StreamWindow.fromElements(11));
+
+		validateOutput(expected2, CentralSink2.windows);
+
+		// groupby mod 3 sum ( Tumbling Count of 2)
+		List<StreamWindow<Integer>> expected3 = new ArrayList<StreamWindow<Integer>>();
+		expected3.add(StreamWindow.fromElements(4));
+		expected3.add(StreamWindow.fromElements(5));
+		expected3.add(StreamWindow.fromElements(16));
+		expected3.add(StreamWindow.fromElements(10));
+		expected3.add(StreamWindow.fromElements(11));
+		expected3.add(StreamWindow.fromElements(3));
+
+		validateOutput(expected3, DistributedSink1.windows);
+
+		// groupby mod3 Tumbling Count of 2 grouped by mod 2
+		List<StreamWindow<Integer>> expected4 = new ArrayList<StreamWindow<Integer>>();
+		expected4.add(StreamWindow.fromElements(2, 2));
+		expected4.add(StreamWindow.fromElements(1));
+		expected4.add(StreamWindow.fromElements(4));
+		expected4.add(StreamWindow.fromElements(5, 11));
+		expected4.add(StreamWindow.fromElements(10));
+		expected4.add(StreamWindow.fromElements(11));
+		expected4.add(StreamWindow.fromElements(3));
+
+		validateOutput(expected4, DistributedSink2.windows);
+
+	}
+
+	private <R> void validateOutput(List<R> expected, List<R> actual) {
+		assertEquals(new HashSet<R>(expected), new HashSet<R>(actual));
+	}
+
+	@SuppressWarnings("serial")
+	private static class CentralSink1 implements SinkFunction<StreamWindow<Integer>> {
+
+		public static List<StreamWindow<Integer>> windows = new ArrayList<StreamWindow<Integer>>();
+
+		@Override
+		public void invoke(StreamWindow<Integer> value) throws Exception {
+			windows.add(value);
+		}
+
+	}
+
+	@SuppressWarnings("serial")
+	private static class CentralSink2 implements SinkFunction<StreamWindow<Integer>> {
+
+		public static List<StreamWindow<Integer>> windows = new ArrayList<StreamWindow<Integer>>();
+
+		@Override
+		public void invoke(StreamWindow<Integer> value) throws Exception {
+			windows.add(value);
+		}
+
+	}
+
+	@SuppressWarnings("serial")
+	private static class DistributedSink1 implements SinkFunction<StreamWindow<Integer>> {
+
+		public static List<StreamWindow<Integer>> windows = new ArrayList<StreamWindow<Integer>>();
+
+		@Override
+		public void invoke(StreamWindow<Integer> value) throws Exception {
+			windows.add(value);
+		}
+
+	}
+
+	@SuppressWarnings("serial")
+	private static class DistributedSink2 implements SinkFunction<StreamWindow<Integer>> {
+
+		public static List<StreamWindow<Integer>> windows = new ArrayList<StreamWindow<Integer>>();
+
+		@Override
+		public void invoke(StreamWindow<Integer> value) throws Exception {
+			windows.add(value);
+		}
+
+	}
+}


[08/20] flink git commit: [streaming] Reimplemented StreamWindow.split to avoid dependency issues

Posted by mb...@apache.org.
[streaming] Reimplemented StreamWindow.split to avoid dependency issues


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cff6552f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cff6552f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cff6552f

Branch: refs/heads/master
Commit: cff6552fff44ff4c8e793efcca5c9b8d87e75e0f
Parents: ef7b7cd
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Feb 11 12:57:06 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../operator/windowing/StreamWindow.java        | 32 ++++++++++++++------
 .../operator/windowing/StreamWindowTest.java    | 30 ++++++++++++++++++
 2 files changed, 52 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/cff6552f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
index 51f2718..90c4e62 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
@@ -27,8 +27,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.util.Collector;
 
-import com.google.common.collect.Lists;
-
 public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 
 	private static final long serialVersionUID = -5150196421193988403L;
@@ -90,15 +88,29 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 	}
 
 	public List<StreamWindow<T>> split(int n) {
-		List<List<T>> subLists = Lists.partition(this, (int) Math.ceil((double) size() / n));
-		List<StreamWindow<T>> split = new ArrayList<StreamWindow<T>>(n);
-		for (List<T> partition : subLists) {
-			StreamWindow<T> subWindow = new StreamWindow<T>(windowID, transformationID,
-					subLists.size());
-			subWindow.addAll(partition);
-			split.add(subWindow);
+		int numElements = size();
+		if (n > numElements) {
+			return split(numElements);
+		} else {
+			List<StreamWindow<T>> split = new ArrayList<StreamWindow<T>>();
+			int splitSize = numElements / n;
+
+			int index = -1;
+
+			StreamWindow<T> currentSubWindow = new StreamWindow<T>(windowID, transformationID, n);
+			split.add(currentSubWindow);
+
+			for (T element : this) {
+				index++;
+				if (index == splitSize && split.size() < n) {
+					currentSubWindow = new StreamWindow<T>(windowID, transformationID, n);
+					split.add(currentSubWindow);
+					index = 0;
+				}
+				currentSubWindow.add(element);
+			}
+			return split;
 		}
-		return split;
 	}
 
 	public StreamWindow<T> setNumberOfParts(int n) {

http://git-wip-us.apache.org/repos/asf/flink/blob/cff6552f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
index 5ceecb1..a66ac77 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
@@ -125,6 +125,36 @@ public class StreamWindowTest {
 
 	@Test
 	public void partitionTest() {
+		StreamWindow<Integer> streamWindow = new StreamWindow<Integer>();
+		streamWindow.add(1);
+		streamWindow.add(2);
+		streamWindow.add(3);
+		streamWindow.add(4);
+		streamWindow.add(5);
+		streamWindow.add(6);
+
+		List<StreamWindow<Integer>> split = streamWindow.split(2);
+		assertEquals(2, split.size());
+		assertEquals(StreamWindow.fromElements(1, 2, 3), split.get(0));
+		assertEquals(StreamWindow.fromElements(4, 5, 6), split.get(1));
+
+		List<StreamWindow<Integer>> split2 = streamWindow.split(6);
+		assertEquals(6, split2.size());
+		assertEquals(StreamWindow.fromElements(1), split2.get(0));
+		assertEquals(StreamWindow.fromElements(2), split2.get(1));
+		assertEquals(StreamWindow.fromElements(3), split2.get(2));
+		assertEquals(StreamWindow.fromElements(4), split2.get(3));
+		assertEquals(StreamWindow.fromElements(5), split2.get(4));
+		assertEquals(StreamWindow.fromElements(6), split2.get(5));
+
+		List<StreamWindow<Integer>> split3 = streamWindow.split(10);
+		assertEquals(6, split3.size());
+		assertEquals(StreamWindow.fromElements(1), split3.get(0));
+		assertEquals(StreamWindow.fromElements(2), split3.get(1));
+		assertEquals(StreamWindow.fromElements(3), split3.get(2));
+		assertEquals(StreamWindow.fromElements(4), split3.get(3));
+		assertEquals(StreamWindow.fromElements(5), split3.get(4));
+		assertEquals(StreamWindow.fromElements(6), split3.get(5));
 
 	}
 


[10/20] flink git commit: [streaming] TumblingGroupedPreReducer added + new tests for windowing

Posted by mb...@apache.org.
[streaming] TumblingGroupedPreReducer added + new tests for windowing


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3885823f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3885823f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3885823f

Branch: refs/heads/master
Commit: 3885823f076eeaeb4d139771c6d3117ed1ba379b
Parents: cff6552
Author: Gyula Fora <gy...@apache.org>
Authored: Thu Feb 12 20:45:34 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../streaming/api/datastream/DataStream.java    |   7 +-
 .../api/datastream/DiscretizedStream.java       |  11 +-
 .../api/datastream/WindowedDataStream.java      |  59 +++++----
 .../windowing/CompletePreAggregator.java        |  23 ++++
 .../windowing/StreamWindowSerializer.java       |   8 +-
 .../windowing/StreamWindowTypeInfo.java         |   6 +-
 .../windowing/TumblingGroupedPreReducer.java    |  95 ++++++++++++++
 .../operator/windowing/TumblingPreReducer.java  |   2 +-
 .../windowing/BasicWindowBufferTest.java        |  83 +++++++++++++
 .../operator/windowing/StreamWindowTest.java    |   2 +-
 .../TumblingGroupedPreReducerTest.java          | 124 +++++++++++++++++++
 .../windowing/TumblingPreReducerTest.java       | 110 ++++++++++++++++
 .../windowing/WindowIntegrationTest.java        |   4 +-
 .../api/scala/WindowedDataStream.scala          |   3 +-
 14 files changed, 498 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index b3d0564..c24ab99 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.commons.lang3.Validate;
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.FilterFunction;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.api.common.functions.MapFunction;
@@ -207,6 +208,10 @@ public class DataStream<OUT> {
 		return environment;
 	}
 
+	public ExecutionConfig getExecutionConfig() {
+		return environment.getConfig();
+	}
+
 	/**
 	 * Creates a new {@link DataStream} by merging {@link DataStream} outputs of
 	 * the same type with each other. The DataStreams merged using this operator
@@ -319,7 +324,7 @@ public class DataStream<OUT> {
 
 	private GroupedDataStream<OUT> groupBy(Keys<OUT> keys) {
 		return new GroupedDataStream<OUT>(this, clean(KeySelectorUtil.getSelectorForKeys(keys,
-				getType(), environment.getConfig())));
+				getType(), getExecutionConfig())));
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
index 5541ba3..c17169d 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
@@ -140,10 +141,7 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 
 			return out;
 		} else if (transformation == WindowTransformation.MAPWINDOW) {
-			return transform(
-					transformation,
-					"Window partitioner",
-					getType(),
+			return transform(transformation, "Window partitioner", getType(),
 					new WindowPartitioner<OUT>(parallelism)).setParallelism(parallelism);
 		} else {
 			return this;
@@ -219,6 +217,11 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		return type;
 	}
 
+	@Override
+	public ExecutionConfig getExecutionConfig() {
+		return discretizedStream.getExecutionConfig();
+	}
+
 	/**
 	 * Gets the output type.
 	 * 

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 8fff958..04a29a9 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
@@ -33,11 +34,13 @@ import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
 import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.invokable.operator.windowing.BasicWindowBuffer;
+import org.apache.flink.streaming.api.invokable.operator.windowing.CompletePreAggregator;
 import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedStreamDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedTimeDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
+import org.apache.flink.streaming.api.invokable.operator.windowing.TumblingGroupedPreReducer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.TumblingPreReducer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowBuffer;
 import org.apache.flink.streaming.api.windowing.helper.Time;
@@ -198,7 +201,8 @@ public class WindowedDataStream<OUT> {
 	}
 
 	private WindowedDataStream<OUT> groupBy(Keys<OUT> keys) {
-		return groupBy(clean(KeySelectorUtil.getSelectorForKeys(keys, getType())));
+		return groupBy(clean(KeySelectorUtil.getSelectorForKeys(keys, getType(),
+				getExecutionConfig())));
 	}
 
 	/**
@@ -214,10 +218,11 @@ public class WindowedDataStream<OUT> {
 		return out;
 	}
 
-	private DiscretizedStream<OUT> discretize(WindowTransformation transformation) {
+	private DiscretizedStream<OUT> discretize(WindowTransformation transformation,
+			WindowBuffer<OUT> windowBuffer) {
 
 		StreamInvokable<OUT, StreamWindow<OUT>> discretizer = getDiscretizer(transformation,
-				getTrigger(), getEviction(), discretizerKey);
+				windowBuffer, getTrigger(), getEviction(), discretizerKey);
 
 		int parallelism = getDiscretizerParallelism();
 
@@ -245,11 +250,9 @@ public class WindowedDataStream<OUT> {
 	}
 
 	private StreamInvokable<OUT, StreamWindow<OUT>> getDiscretizer(
-			WindowTransformation transformation, TriggerPolicy<OUT> trigger,
-			EvictionPolicy<OUT> eviction, KeySelector<OUT, ?> discretizerKey) {
-
-		WindowBuffer<OUT> windowBuffer = getWindowBuffer(transformation, trigger, eviction,
-				discretizerKey);
+			WindowTransformation transformation, WindowBuffer<OUT> windowBuffer,
+			TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> eviction,
+			KeySelector<OUT, ?> discretizerKey) {
 
 		if (discretizerKey == null) {
 			return new StreamDiscretizer<OUT>(trigger, eviction, windowBuffer);
@@ -273,11 +276,12 @@ public class WindowedDataStream<OUT> {
 
 		if (transformation == WindowTransformation.REDUCEWINDOW
 				&& eviction instanceof TumblingEvictionPolicy) {
-			if (discretizerKey == null) {
+			if (groupByKey == null) {
 				return new TumblingPreReducer<OUT>((ReduceFunction<OUT>) transformation.UDF,
-						getType().createSerializer());
+						getType().createSerializer(getExecutionConfig()));
 			} else {
-				return new BasicWindowBuffer<OUT>();
+				return new TumblingGroupedPreReducer<OUT>((ReduceFunction<OUT>) transformation.UDF,
+						groupByKey, getType().createSerializer(getExecutionConfig()));
 			}
 		}
 		return new BasicWindowBuffer<OUT>();
@@ -294,8 +298,18 @@ public class WindowedDataStream<OUT> {
 	 * @return The transformed DataStream
 	 */
 	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
-		return discretize(WindowTransformation.REDUCEWINDOW.with(reduceFunction)).reduceWindow(
-				reduceFunction);
+
+		WindowTransformation transformation = WindowTransformation.REDUCEWINDOW
+				.with(reduceFunction);
+
+		WindowBuffer<OUT> windowBuffer = getWindowBuffer(transformation, getTrigger(),
+				getEviction(), discretizerKey);
+
+		if (windowBuffer instanceof CompletePreAggregator) {
+			return discretize(transformation, windowBuffer);
+		} else {
+			return discretize(transformation, windowBuffer).reduceWindow(reduceFunction);
+		}
 	}
 
 	/**
@@ -312,8 +326,8 @@ public class WindowedDataStream<OUT> {
 	 * @return The transformed DataStream
 	 */
 	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction) {
-		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction)).mapWindow(
-				reduceFunction);
+		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction),
+				new BasicWindowBuffer<OUT>()).mapWindow(reduceFunction);
 	}
 
 	/**
@@ -335,8 +349,8 @@ public class WindowedDataStream<OUT> {
 	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction,
 			TypeInformation<R> outType) {
 
-		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction)).mapWindow(
-				reduceFunction, outType);
+		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction),
+				new BasicWindowBuffer<OUT>()).mapWindow(reduceFunction, outType);
 	}
 
 	public DataStream<OUT> flatten() {
@@ -605,10 +619,6 @@ public class WindowedDataStream<OUT> {
 		return dataStream.getType();
 	}
 
-	protected DataStream<OUT> getDataStream() {
-		return dataStream;
-	}
-
 	protected WindowedDataStream<OUT> copy() {
 		return new WindowedDataStream<OUT>(this);
 	}
@@ -618,7 +628,12 @@ public class WindowedDataStream<OUT> {
 	}
 
 	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
-		return discretize(WindowTransformation.NONE).getDiscretizedStream();
+		return discretize(WindowTransformation.NONE, new BasicWindowBuffer<OUT>())
+				.getDiscretizedStream();
+	}
+
+	public ExecutionConfig getExecutionConfig() {
+		return dataStream.getExecutionConfig();
 	}
 
 	protected static class WindowKey<R> implements KeySelector<StreamWindow<R>, Integer> {

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java
new file mode 100644
index 0000000..e071c2b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+public interface CompletePreAggregator {
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
index 12e3574..002e440 100755
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import java.io.IOException;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -31,10 +32,11 @@ public final class StreamWindowSerializer<T> extends TypeSerializer<StreamWindow
 	private static final long serialVersionUID = 1L;
 
 	private final TypeSerializer<T> typeSerializer;
-	TypeSerializer<Integer> intSerializer = BasicTypeInfo.INT_TYPE_INFO.createSerializer();
+	TypeSerializer<Integer> intSerializer;
 
-	public StreamWindowSerializer(TypeInformation<T> typeInfo) {
-		this.typeSerializer = typeInfo.createSerializer();
+	public StreamWindowSerializer(TypeInformation<T> typeInfo, ExecutionConfig conf) {
+		this.typeSerializer = typeInfo.createSerializer(conf);
+		this.intSerializer = BasicTypeInfo.INT_TYPE_INFO.createSerializer(conf);
 	}
 
 	public TypeSerializer<T> getObjectSerializer() {

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
index a9a4bba..0054759 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
@@ -18,11 +18,13 @@
 
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 
 public class StreamWindowTypeInfo<T> extends TypeInformation<StreamWindow<T>> {
 
+	private static final long serialVersionUID = 1L;
 	TypeInformation<T> innerType;
 
 	public StreamWindowTypeInfo(TypeInformation<T> innerType) {
@@ -60,8 +62,8 @@ public class StreamWindowTypeInfo<T> extends TypeInformation<StreamWindow<T>> {
 	}
 
 	@Override
-	public TypeSerializer<StreamWindow<T>> createSerializer() {
-		return new StreamWindowSerializer<T>(innerType);
+	public TypeSerializer<StreamWindow<T>> createSerializer(ExecutionConfig conf) {
+		return new StreamWindowSerializer<T>(innerType, conf);
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java
new file mode 100644
index 0000000..f1d7ae4
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.util.Collector;
+
+public class TumblingGroupedPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
+
+	private static final long serialVersionUID = 1L;
+
+	private ReduceFunction<T> reducer;
+	private KeySelector<T, ?> keySelector;
+
+	private Map<Object, T> reducedValues;
+
+	private int numOfElements = 0;
+	private TypeSerializer<T> serializer;
+
+	public TumblingGroupedPreReducer(ReduceFunction<T> reducer, KeySelector<T, ?> keySelector,
+			TypeSerializer<T> serializer) {
+		this.reducer = reducer;
+		this.serializer = serializer;
+		this.keySelector = keySelector;
+		this.reducedValues = new HashMap<Object, T>();
+	}
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
+
+		if (!reducedValues.isEmpty()) {
+			StreamWindow<T> currentWindow = new StreamWindow<T>();
+			currentWindow.addAll(reducedValues.values());
+			collector.collect(currentWindow);
+			reducedValues.clear();
+			numOfElements = 0;
+			return true;
+		} else {
+			return false;
+		}
+
+	}
+
+	public void store(T element) throws Exception {
+		Object key = keySelector.getKey(element);
+
+		T reduced = reducedValues.get(key);
+
+		if (reduced == null) {
+			reduced = element;
+		} else {
+			reduced = reducer.reduce(serializer.copy(reduced), element);
+		}
+
+		reducedValues.put(key, reduced);
+		numOfElements++;
+	}
+
+	public void evict(int n) {
+	}
+
+	public int size() {
+		return numOfElements;
+	}
+
+	@Override
+	public TumblingGroupedPreReducer<T> clone() {
+		return new TumblingGroupedPreReducer<T>(reducer, keySelector, serializer);
+	}
+
+	@Override
+	public String toString() {
+		return reducedValues.toString();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
index 4f05813..f1e531e 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
@@ -21,7 +21,7 @@ import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.util.Collector;
 
-public class TumblingPreReducer<T> implements WindowBuffer<T> {
+public class TumblingPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
 
 	private static final long serialVersionUID = 1L;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java
new file mode 100644
index 0000000..e079e2c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class BasicWindowBufferTest {
+
+	@Test
+	public void testEmitWindow() throws Exception {
+
+		TestCollector<StreamWindow<Integer>> collector = new TestCollector<StreamWindow<Integer>>();
+		List<StreamWindow<Integer>> collected = collector.getCollected();
+
+		WindowBuffer<Integer> wb = new BasicWindowBuffer<Integer>();
+
+		wb.store(2);
+		wb.store(10);
+
+		assertEquals(2, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(1, collected.size());
+		assertEquals(StreamWindow.fromElements(2, 10), collected.get(0));
+
+		wb.store(4);
+		wb.evict(2);
+
+		assertEquals(1, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(2, collected.size());
+		assertEquals(StreamWindow.fromElements(4), collected.get(1));
+
+		wb.evict(1);
+
+		wb.emitWindow(collector);
+		assertEquals(2, collected.size());
+	}
+
+	public static class TestCollector<T> implements Collector<T> {
+
+		private final List<T> collected = new ArrayList<T>();
+
+		@Override
+		public void collect(T record) {
+			collected.add(record);
+		}
+
+		@Override
+		public void close() {
+		}
+
+		public List<T> getCollected() {
+			return collected;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
index a66ac77..88e90c9 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
@@ -111,7 +111,7 @@ public class StreamWindowTest {
 		streamWindow.add(3);
 
 		TypeSerializer<StreamWindow<Integer>> ts = new StreamWindowSerializer<Integer>(
-				BasicTypeInfo.INT_TYPE_INFO);
+				BasicTypeInfo.INT_TYPE_INFO, null);
 
 		TestOutputView ow = new TestOutputView();
 

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java
new file mode 100644
index 0000000..8287202
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.invokable.operator.windowing.BasicWindowBufferTest.TestCollector;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
+import org.junit.Test;
+
+public class TumblingGroupedPreReducerTest {
+
+	TypeInformation<Tuple2<Integer, Integer>> type = TypeExtractor
+			.getForObject(new Tuple2<Integer, Integer>(1, 1));
+	TypeSerializer<Tuple2<Integer, Integer>> serializer = type.createSerializer(null);
+
+	KeySelector<Tuple2<Integer, Integer>, ?> key = KeySelectorUtil.getSelectorForKeys(
+			new Keys.ExpressionKeys<Tuple2<Integer, Integer>>(new int[] { 0 }, type), type, null);
+
+	Reducer reducer = new Reducer();
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testEmitWindow() throws Exception {
+
+		List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
+		inputs.add(new Tuple2<Integer, Integer>(1, 1));
+		inputs.add(new Tuple2<Integer, Integer>(0, 0));
+		inputs.add(new Tuple2<Integer, Integer>(1, -1));
+		inputs.add(new Tuple2<Integer, Integer>(1, -2));
+
+		TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector = new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
+		List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
+
+		WindowBuffer<Tuple2<Integer, Integer>> wb = new TumblingGroupedPreReducer<Tuple2<Integer, Integer>>(
+				reducer, key, serializer);
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+
+		assertEquals(2, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(1, collected.size());
+
+		assertSetEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(1, 1),
+				new Tuple2<Integer, Integer>(0, 0)), collected.get(0));
+
+		// Test automatic eviction
+		assertEquals(0, wb.size());
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+		wb.store(serializer.copy(inputs.get(2)));
+
+		// Nothing should happen here
+		wb.evict(3);
+
+		wb.store(serializer.copy(inputs.get(3)));
+
+		assertEquals(4, wb.size());
+		wb.emitWindow(collector);
+
+		assertEquals(2, collected.size());
+
+		assertSetEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(3, -2),
+				new Tuple2<Integer, Integer>(0, 0)), collected.get(1));
+
+		// Test whether function is mutating inputs or not
+		assertEquals(2, reducer.allInputs.size());
+		assertEquals(reducer.allInputs.get(0), inputs.get(2));
+		assertEquals(reducer.allInputs.get(1), inputs.get(3));
+
+	}
+
+	private static <T> void assertSetEquals(Collection<T> first, Collection<T> second) {
+		assertEquals(new HashSet<T>(first), new HashSet<T>(second));
+	}
+
+	@SuppressWarnings("serial")
+	private class Reducer implements ReduceFunction<Tuple2<Integer, Integer>> {
+
+		public List<Tuple2<Integer, Integer>> allInputs = new ArrayList<Tuple2<Integer, Integer>>();
+
+		@Override
+		public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1,
+				Tuple2<Integer, Integer> value2) throws Exception {
+			allInputs.add(value2);
+			value1.f0 = value1.f0 + value2.f0;
+			value1.f1 = value1.f1 + value2.f1;
+			return value1;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java
new file mode 100644
index 0000000..6e29e68
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.invokable.operator.windowing.BasicWindowBufferTest.TestCollector;
+import org.junit.Test;
+
+public class TumblingPreReducerTest {
+
+	TypeSerializer<Tuple2<Integer, Integer>> serializer = TypeExtractor.getForObject(
+			new Tuple2<Integer, Integer>(1, 1)).createSerializer(null);
+
+	Reducer reducer = new Reducer();
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void testEmitWindow() throws Exception {
+
+		List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
+		inputs.add(new Tuple2<Integer, Integer>(1, 1));
+		inputs.add(new Tuple2<Integer, Integer>(2, 0));
+		inputs.add(new Tuple2<Integer, Integer>(3, -1));
+		inputs.add(new Tuple2<Integer, Integer>(4, -2));
+
+		TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector = new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
+		List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
+
+		WindowBuffer<Tuple2<Integer, Integer>> wb = new TumblingPreReducer<Tuple2<Integer, Integer>>(
+				reducer, serializer);
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+
+		assertEquals(2, wb.size());
+
+		wb.emitWindow(collector);
+
+		assertEquals(1, collected.size());
+		assertEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(3, 1)),
+				collected.get(0));
+
+		// Test automatic eviction
+		assertEquals(0, wb.size());
+
+		wb.store(serializer.copy(inputs.get(0)));
+		wb.store(serializer.copy(inputs.get(1)));
+		wb.store(serializer.copy(inputs.get(2)));
+
+		// Nothing should happen here
+		wb.evict(3);
+
+		wb.store(serializer.copy(inputs.get(3)));
+
+		assertEquals(4, wb.size());
+		wb.emitWindow(collector);
+
+		assertEquals(2, collected.size());
+		assertEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(10, -2)),
+				collected.get(1));
+
+		// Test whether function is mutating inputs or not
+		assertEquals(4, reducer.allInputs.size());
+		assertEquals(reducer.allInputs.get(0), inputs.get(1));
+		assertEquals(reducer.allInputs.get(1), inputs.get(1));
+		assertEquals(reducer.allInputs.get(2), inputs.get(2));
+		assertEquals(reducer.allInputs.get(3), inputs.get(3));
+
+	}
+
+	@SuppressWarnings("serial")
+	private class Reducer implements ReduceFunction<Tuple2<Integer, Integer>> {
+
+		public List<Tuple2<Integer, Integer>> allInputs = new ArrayList<Tuple2<Integer, Integer>>();
+
+		@Override
+		public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1,
+				Tuple2<Integer, Integer> value2) throws Exception {
+			allInputs.add(value2);
+			value1.f0 = value1.f0 + value2.f0;
+			value1.f1 = value1.f1 + value2.f1;
+			return value1;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
index bcde0c5..ed6b347 100755
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
@@ -54,7 +54,7 @@ public class WindowIntegrationTest implements Serializable {
 	}
 
 	@SuppressWarnings("serial")
-	private static class IdentityWindowMap implements
+	public static class IdentityWindowMap implements
 			GroupReduceFunction<Integer, StreamWindow<Integer>> {
 
 		@Override
@@ -101,8 +101,6 @@ public class WindowIntegrationTest implements Serializable {
 		source.groupBy(new ModKey(3)).window(Count.of(2)).groupBy(new ModKey(2))
 				.mapWindow(new IdentityWindowMap()).flatten().addSink(new DistributedSink2());
 
-		env.generateSequence(1, 10).window(Count.of(3)).sum(0).getDiscretizedStream().print();
-		
 		env.execute();
 
 		// sum ( Count of 2 slide 3 )

http://git-wip-us.apache.org/repos/asf/flink/blob/3885823f/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
index 5f244d0..780d703 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -257,8 +257,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
     val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
 
     val agg = new ScalaStreamingAggregator[Product](
-      jStream.getType().createSerializer(
-        javaStream.getDataStream.getExecutionEnvironment.getConfig),
+      jStream.getType().createSerializer(javaStream.getExecutionConfig),
       position)
 
     val reducer = aggregationType match {


[07/20] flink git commit: [FLINK-1231] [streaming] More sophisticated streaming classloader ITCase

Posted by mb...@apache.org.
[FLINK-1231] [streaming] More sophisticated streaming classloader ITCase

This closes #303


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/26ae9793
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/26ae9793
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/26ae9793

Branch: refs/heads/master
Commit: 26ae9793cf6dbccdfb525637ea6caffa0f1712e9
Parents: 055f6dc
Author: mbalassi <mb...@apache.org>
Authored: Fri Feb 13 15:57:39 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:07 2015 +0100

----------------------------------------------------------------------
 .../test/classloading/jar/StreamingProgram.java | 73 +++++++++++++++-----
 1 file changed, 57 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/26ae9793/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java
index 6799d9b..f7f9eae 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/jar/StreamingProgram.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.streaming.api.function.sink.SinkFunction;
+import org.apache.flink.test.recordJobs.wordcount.WordCount;
 import org.apache.flink.test.testdata.WordCountData;
 import org.apache.flink.util.Collector;
 
@@ -41,23 +42,63 @@ public class StreamingProgram {
 
 		DataStream<String> text = env.fromElements(WordCountData.TEXT);
 
-		DataStream<Tuple2<String, Integer>> counts =
-				text.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
-					@Override
-					public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
-						StringTokenizer tokenizer = new StringTokenizer(value);
-						while (tokenizer.hasMoreTokens()){
-							out.collect(new Tuple2<String, Integer>(tokenizer.nextToken(), 1));
-						}
-					}
-				}).groupBy(0).sum(1);
-
-		counts.addSink(new SinkFunction<Tuple2<String, Integer>>() {
-			@Override
-			public void invoke(Tuple2<String, Integer> value) throws Exception {
-			}
-		});
+		DataStream<Word> counts =
+				text.flatMap(new Tokenizer()).groupBy("word").sum("frequency");
+
+		counts.addSink(new NoOpSink());
 
 		env.execute();
 	}
+	// --------------------------------------------------------------------------------------------
+
+	public static class Word {
+
+		private String word;
+		private Integer frequency;
+
+		public Word() {
+		}
+
+		public Word(String word, int i) {
+			this.word = word;
+			this.frequency = i;
+		}
+
+		public String getWord() {
+			return word;
+		}
+
+		public void setWord(String word) {
+			this.word = word;
+		}
+
+		public Integer getFrequency() {
+			return frequency;
+		}
+
+		public void setFrequency(Integer frequency) {
+			this.frequency = frequency;
+		}
+
+		@Override
+		public String toString() {
+			return "(" + word + ", " + frequency + ")";
+		}
+	}
+
+	public static class Tokenizer implements FlatMapFunction<String, Word>{
+		@Override
+		public void flatMap(String value, Collector<Word> out) throws Exception {
+			StringTokenizer tokenizer = new StringTokenizer(value);
+			while (tokenizer.hasMoreTokens()){
+				out.collect(new Word(tokenizer.nextToken(), 1));
+			}
+		}
+	}
+
+	public static class NoOpSink implements SinkFunction<Word>{
+		@Override
+		public void invoke(Word value) throws Exception {
+		}
+	}
 }


[17/20] flink git commit: [streaming] WindowMapFunction added + Streaming package structure cleanup

Posted by mb...@apache.org.
[streaming] WindowMapFunction added + Streaming package structure cleanup


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bb5dc7e3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bb5dc7e3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bb5dc7e3

Branch: refs/heads/master
Commit: bb5dc7e3dd6825f8f78f42df6c4ca5ef00d520fe
Parents: 3885823
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Feb 13 11:03:08 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../flink/api/java/typeutils/TypeExtractor.java |   2 +-
 .../streaming/api/datastream/DataStream.java    |  16 +-
 .../api/datastream/DiscretizedStream.java       |  55 ++---
 .../api/datastream/WindowedDataStream.java      | 218 +++++++++----------
 .../temporaloperator/StreamCrossOperator.java   |  13 +-
 .../environment/StreamExecutionEnvironment.java |   1 +
 .../api/function/RichWindowMapFunction.java     |  31 +++
 .../api/function/WindowMapFunction.java         |  28 +++
 .../function/source/ParallelSourceFunction.java |  40 ++--
 .../operator/windowing/BasicWindowBuffer.java   |  74 -------
 .../windowing/CompletePreAggregator.java        |  23 --
 .../windowing/GroupedStreamDiscretizer.java     |   2 +
 .../windowing/GroupedTimeDiscretizer.java       |   1 +
 .../operator/windowing/StreamDiscretizer.java   |   2 +
 .../operator/windowing/StreamWindow.java        | 183 ----------------
 .../windowing/StreamWindowSerializer.java       | 138 ------------
 .../windowing/StreamWindowTypeInfo.java         |  74 -------
 .../windowing/TumblingGroupedPreReducer.java    |  95 --------
 .../operator/windowing/TumblingPreReducer.java  |  78 -------
 .../operator/windowing/WindowBuffer.java        |  36 ---
 .../operator/windowing/WindowFlattener.java     |   1 +
 .../operator/windowing/WindowMapper.java        |  22 +-
 .../operator/windowing/WindowMerger.java        |   1 +
 .../operator/windowing/WindowPartitioner.java   |   1 +
 .../operator/windowing/WindowReducer.java       |   1 +
 .../streaming/api/windowing/StreamWindow.java   | 181 +++++++++++++++
 .../api/windowing/StreamWindowSerializer.java   | 134 ++++++++++++
 .../api/windowing/StreamWindowTypeInfo.java     |  74 +++++++
 .../windowbuffer/BasicWindowBuffer.java         |  75 +++++++
 .../windowbuffer/CompletePreAggregator.java     |  23 ++
 .../windowbuffer/TumblingGroupedPreReducer.java |  96 ++++++++
 .../windowbuffer/TumblingPreReducer.java        |  79 +++++++
 .../windowing/windowbuffer/WindowBuffer.java    |  37 ++++
 .../api/invokable/operator/CoFlatMapTest.java   |  84 -------
 .../invokable/operator/CoGroupedReduceTest.java | 125 -----------
 .../api/invokable/operator/CoMapTest.java       |  57 -----
 .../invokable/operator/CoStreamReduceTest.java  |  71 ------
 .../api/invokable/operator/CoWindowTest.java    | 182 ----------------
 .../invokable/operator/co/CoFlatMapTest.java    |  84 +++++++
 .../operator/co/CoGroupedReduceTest.java        | 125 +++++++++++
 .../api/invokable/operator/co/CoMapTest.java    |  57 +++++
 .../operator/co/CoStreamReduceTest.java         |  71 ++++++
 .../api/invokable/operator/co/CoWindowTest.java | 182 ++++++++++++++++
 .../windowing/BasicWindowBufferTest.java        |  83 -------
 .../windowing/GroupedStreamDiscretizerTest.java |   2 +
 .../windowing/StreamDiscretizerTest.java        |   2 +
 .../operator/windowing/StreamWindowTest.java    | 201 -----------------
 .../TumblingGroupedPreReducerTest.java          | 124 -----------
 .../windowing/TumblingPreReducerTest.java       | 110 ----------
 .../windowing/WindowIntegrationTest.java        |   7 +-
 .../api/windowing/StreamWindowTest.java         | 201 +++++++++++++++++
 .../windowbuffer/BasicWindowBufferTest.java     |  86 ++++++++
 .../TumblingGroupedPreReducerTest.java          | 127 +++++++++++
 .../windowbuffer/TumblingPreReducerTest.java    | 113 ++++++++++
 .../ml/IncrementalLearningSkeleton.java         |   6 +-
 .../examples/windowing/StockPrices.java         |  30 +--
 .../api/scala/WindowedDataStream.scala          |  14 +-
 .../flink/streaming/api/scala/package.scala     |   2 +-
 58 files changed, 2032 insertions(+), 1949 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 1b3003f..63273f8 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -212,7 +212,7 @@ public class TypeExtractor {
 	// --------------------------------------------------------------------------------------------
 	
 	@SuppressWarnings("unchecked")
-	private static <IN, OUT> TypeInformation<OUT> getUnaryOperatorReturnType(Function function, Class<?> baseClass, 
+	public static <IN, OUT> TypeInformation<OUT> getUnaryOperatorReturnType(Function function, Class<?> baseClass, 
 			boolean hasIterable, boolean hasCollector, TypeInformation<IN> inType,
 			String functionName, boolean allowMissing)
 	{

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index c24ab99..e766626 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -208,7 +208,7 @@ public class DataStream<OUT> {
 		return environment;
 	}
 
-	public ExecutionConfig getExecutionConfig() {
+	protected ExecutionConfig getExecutionConfig() {
 		return environment.getConfig();
 	}
 
@@ -254,11 +254,12 @@ public class DataStream<OUT> {
 
 	/**
 	 * Creates a new {@link ConnectedDataStream} by connecting
-	 * {@link DataStream} outputs of different type with each other. The
-	 * DataStreams connected using this operators can be used with CoFunctions.
+	 * {@link DataStream} outputs of (possible) different typea with each other.
+	 * The DataStreams connected using this operator can be used with
+	 * CoFunctions to apply joint transformations.
 	 * 
 	 * @param dataStream
-	 *            The DataStream with which this stream will be joined.
+	 *            The DataStream with which this stream will be connected.
 	 * @return The {@link ConnectedDataStream}.
 	 */
 	public <R> ConnectedDataStream<OUT, R> connect(DataStream<R> dataStream) {
@@ -502,9 +503,10 @@ public class DataStream<OUT> {
 	}
 
 	/**
-	 * Applies a reduce transformation on the data stream. The user can also
-	 * extend the {@link RichReduceFunction} to gain access to other features
-	 * provided by the
+	 * Applies a reduce transformation on the data stream. The returned stream
+	 * contains all the intermediate values of the reduce transformation. The
+	 * user can also extend the {@link RichReduceFunction} to gain access to
+	 * other features provided by the
 	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
 	 * 
 	 * @param reducer

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
index c17169d..7ab7c2c 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
@@ -18,9 +18,8 @@
 package org.apache.flink.streaming.api.datastream;
 
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
 import org.apache.flink.api.common.functions.RichReduceFunction;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
@@ -28,21 +27,21 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
-import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowFlattener;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowMapper;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowMerger;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowPartitioner;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowReducer;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.api.windowing.StreamWindowTypeInfo;
 
 /**
  * A {@link DiscretizedStream} represents a data stream that has been divided
  * into windows (predefined chunks). User defined function such as
- * {@link #reduceWindow(ReduceFunction)},
- * {@link #mapWindow(GroupReduceFunction)} or aggregations can be applied to the
- * windows.
+ * {@link #reduceWindow(ReduceFunction)}, {@link #mapWindow()} or aggregations
+ * can be applied to the windows.
  * 
  * @param <OUT>
  *            The output type of the {@link DiscretizedStream}
@@ -60,6 +59,14 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		this.transformation = tranformation;
 	}
 
+	public DataStream<OUT> flatten() {
+		return discretizedStream.transform("Window Flatten", getType(), new WindowFlattener<OUT>());
+	}
+
+	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
+		return discretizedStream;
+	}
+
 	/**
 	 * Applies a reduce transformation on the windowed data stream by reducing
 	 * the current window at every trigger.The user can also extend the
@@ -79,7 +86,7 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 
 		if (!isGrouped()) {
 			return out.transform(WindowTransformation.REDUCEWINDOW, "Window Reduce", out.getType(),
-					new WindowReducer<OUT>(reduceFunction));
+					new WindowReducer<OUT>(discretizedStream.clean(reduceFunction)));
 		} else {
 			return out;
 		}
@@ -90,29 +97,27 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 	 * reducing the current window at every trigger. In contrast with the
 	 * standard binary reducer, with reduceGroup the user can access all
 	 * elements of the window at the same time through the iterable interface.
-	 * The user can also extend the {@link RichGroupReduceFunction} to gain
-	 * access to other features provided by the
-	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+	 * The user can also extend the to gain access to other features provided by
+	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
 	 * 
-	 * @param reduceFunction
+	 * @param windowMapFunction
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
 	@Override
-	public <R> DiscretizedStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction) {
+	public <R> DiscretizedStream<R> mapWindow(WindowMapFunction<OUT, R> windowMapFunction) {
 
-		TypeInformation<R> retType = TypeExtractor.getGroupReduceReturnTypes(reduceFunction,
-				getType());
+		TypeInformation<R> retType = getWindowMapReturnTypes(windowMapFunction, getType());
 
-		return mapWindow(reduceFunction, retType);
+		return mapWindow(windowMapFunction, retType);
 	}
 
 	@Override
-	public <R> DiscretizedStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction,
+	public <R> DiscretizedStream<R> mapWindow(WindowMapFunction<OUT, R> windowMapFunction,
 			TypeInformation<R> returnType) {
 		DiscretizedStream<R> out = partition(transformation).transform(
 				WindowTransformation.REDUCEWINDOW, "Window Reduce", returnType,
-				new WindowMapper<OUT, R>(reduceFunction));
+				new WindowMapper<OUT, R>(discretizedStream.clean(windowMapFunction)));
 
 		if (isGrouped()) {
 			return out.merge();
@@ -159,10 +164,6 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 				type, new WindowMerger<OUT>()));
 	}
 
-	public DataStream<OUT> flatten() {
-		return discretizedStream.transform("Window Flatten", getType(), new WindowFlattener<OUT>());
-	}
-
 	@SuppressWarnings("rawtypes")
 	private <R> DiscretizedStream<R> wrap(SingleOutputStreamOperator stream) {
 		return wrap(stream, transformation);
@@ -174,10 +175,6 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		return new DiscretizedStream<R>(stream, (KeySelector<R, ?>) this.groupByKey, transformation);
 	}
 
-	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
-		return discretizedStream;
-	}
-
 	@SuppressWarnings("rawtypes")
 	protected Class<?> getClassAtPos(int pos) {
 		Class<?> type;
@@ -231,6 +228,12 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		return ((StreamWindowTypeInfo<OUT>) discretizedStream.getType()).getInnerType();
 	}
 
+	private static <IN, OUT> TypeInformation<OUT> getWindowMapReturnTypes(
+			WindowMapFunction<IN, OUT> windowMapInterface, TypeInformation<IN> inType) {
+		return TypeExtractor.getUnaryOperatorReturnType((Function) windowMapInterface,
+				WindowMapFunction.class, true, true, inType, null, false);
+	}
+
 	protected DiscretizedStream<OUT> copy() {
 		return new DiscretizedStream<OUT>(discretizedStream.copy(), groupByKey, transformation);
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 04a29a9..3ff5859 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -19,30 +19,24 @@ package org.apache.flink.streaming.api.datastream;
 
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichGroupReduceFunction;
 import org.apache.flink.api.common.functions.RichReduceFunction;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
 import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
 import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.windowing.BasicWindowBuffer;
-import org.apache.flink.streaming.api.invokable.operator.windowing.CompletePreAggregator;
 import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedStreamDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedTimeDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamDiscretizer;
-import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
-import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
-import org.apache.flink.streaming.api.invokable.operator.windowing.TumblingGroupedPreReducer;
-import org.apache.flink.streaming.api.invokable.operator.windowing.TumblingPreReducer;
-import org.apache.flink.streaming.api.invokable.operator.windowing.WindowBuffer;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.api.windowing.StreamWindowTypeInfo;
 import org.apache.flink.streaming.api.windowing.helper.Time;
 import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
@@ -51,20 +45,36 @@ import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.windowbuffer.BasicWindowBuffer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.CompletePreAggregator;
+import org.apache.flink.streaming.api.windowing.windowbuffer.TumblingGroupedPreReducer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.TumblingPreReducer;
+import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 import org.apache.flink.streaming.util.keys.KeySelectorUtil;
 
 /**
  * A {@link WindowedDataStream} represents a data stream that has been divided
  * into windows (predefined chunks). User defined function such as
- * {@link #reduceWindow(ReduceFunction)},
- * {@link #mapWindow(GroupReduceFunction)} or aggregations can be applied to the
- * windows.
+ * {@link #reduceWindow(ReduceFunction)}, {@link #mapWindow()} or aggregations
+ * can be applied to the windows.
  * 
  * @param <OUT>
  *            The output type of the {@link WindowedDataStream}
  */
 public class WindowedDataStream<OUT> {
 
+	protected enum WindowTransformation {
+
+		REDUCEWINDOW, MAPWINDOW, NONE;
+
+		private Function UDF;
+
+		public WindowTransformation with(Function UDF) {
+			this.UDF = UDF;
+			return this;
+		}
+	}
+
 	protected DataStream<OUT> dataStream;
 
 	protected boolean isLocal = false;
@@ -113,10 +123,6 @@ public class WindowedDataStream<OUT> {
 	public WindowedDataStream() {
 	}
 
-	public <F> F clean(F f) {
-		return dataStream.clean(f);
-	}
-
 	/**
 	 * Defines the slide size (trigger frequency) for the windowed data stream.
 	 * This controls how often the user defined function will be triggered on
@@ -201,7 +207,7 @@ public class WindowedDataStream<OUT> {
 	}
 
 	private WindowedDataStream<OUT> groupBy(Keys<OUT> keys) {
-		return groupBy(clean(KeySelectorUtil.getSelectorForKeys(keys, getType(),
+		return groupBy(dataStream.clean(KeySelectorUtil.getSelectorForKeys(keys, getType(),
 				getExecutionConfig())));
 	}
 
@@ -218,73 +224,13 @@ public class WindowedDataStream<OUT> {
 		return out;
 	}
 
-	private DiscretizedStream<OUT> discretize(WindowTransformation transformation,
-			WindowBuffer<OUT> windowBuffer) {
-
-		StreamInvokable<OUT, StreamWindow<OUT>> discretizer = getDiscretizer(transformation,
-				windowBuffer, getTrigger(), getEviction(), discretizerKey);
-
-		int parallelism = getDiscretizerParallelism();
-
-		return new DiscretizedStream<OUT>(dataStream.transform("Stream Discretizer",
-				new StreamWindowTypeInfo<OUT>(getType()), discretizer).setParallelism(parallelism),
-				groupByKey, transformation);
-
-	}
-
-	protected enum WindowTransformation {
-
-		REDUCEWINDOW, MAPWINDOW, NONE;
-
-		private Function UDF;
-
-		public WindowTransformation with(Function UDF) {
-			this.UDF = UDF;
-			return this;
-		}
-	}
-
-	private int getDiscretizerParallelism() {
-		return isLocal || (discretizerKey != null) ? dataStream.environment
-				.getDegreeOfParallelism() : 1;
-	}
-
-	private StreamInvokable<OUT, StreamWindow<OUT>> getDiscretizer(
-			WindowTransformation transformation, WindowBuffer<OUT> windowBuffer,
-			TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> eviction,
-			KeySelector<OUT, ?> discretizerKey) {
-
-		if (discretizerKey == null) {
-			return new StreamDiscretizer<OUT>(trigger, eviction, windowBuffer);
-		} else if (trigger instanceof TimeTriggerPolicy
-				&& ((TimeTriggerPolicy<OUT>) trigger).timestampWrapper.isDefaultTimestamp()) {
-			return new GroupedTimeDiscretizer<OUT>(discretizerKey,
-					(TimeTriggerPolicy<OUT>) trigger, (CloneableEvictionPolicy<OUT>) eviction,
-					windowBuffer);
-		} else {
-			return new GroupedStreamDiscretizer<OUT>(discretizerKey,
-					(CloneableTriggerPolicy<OUT>) trigger, (CloneableEvictionPolicy<OUT>) eviction,
-					windowBuffer);
-		}
-
+	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
+		return discretize(WindowTransformation.NONE, new BasicWindowBuffer<OUT>())
+				.getDiscretizedStream();
 	}
 
-	@SuppressWarnings("unchecked")
-	private WindowBuffer<OUT> getWindowBuffer(WindowTransformation transformation,
-			TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> eviction,
-			KeySelector<OUT, ?> discretizerKey) {
-
-		if (transformation == WindowTransformation.REDUCEWINDOW
-				&& eviction instanceof TumblingEvictionPolicy) {
-			if (groupByKey == null) {
-				return new TumblingPreReducer<OUT>((ReduceFunction<OUT>) transformation.UDF,
-						getType().createSerializer(getExecutionConfig()));
-			} else {
-				return new TumblingGroupedPreReducer<OUT>((ReduceFunction<OUT>) transformation.UDF,
-						groupByKey, getType().createSerializer(getExecutionConfig()));
-			}
-		}
-		return new BasicWindowBuffer<OUT>();
+	public DataStream<OUT> flatten() {
+		return dataStream;
 	}
 
 	/**
@@ -317,17 +263,16 @@ public class WindowedDataStream<OUT> {
 	 * reducing the current window at every trigger. In contrast with the
 	 * standard binary reducer, with reduceGroup the user can access all
 	 * elements of the window at the same time through the iterable interface.
-	 * The user can also extend the {@link RichGroupReduceFunction} to gain
-	 * access to other features provided by the
-	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+	 * The user can also extend the to gain access to other features provided by
+	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
 	 * 
-	 * @param reduceFunction
+	 * @param windowMapFunction
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
-	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction) {
-		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction),
-				new BasicWindowBuffer<OUT>()).mapWindow(reduceFunction);
+	public <R> WindowedDataStream<R> mapWindow(WindowMapFunction<OUT, R> windowMapFunction) {
+		return discretize(WindowTransformation.MAPWINDOW.with(windowMapFunction),
+				new BasicWindowBuffer<OUT>()).mapWindow(windowMapFunction);
 	}
 
 	/**
@@ -335,30 +280,80 @@ public class WindowedDataStream<OUT> {
 	 * reducing the current window at every trigger. In contrast with the
 	 * standard binary reducer, with reduceGroup the user can access all
 	 * elements of the window at the same time through the iterable interface.
-	 * The user can also extend the {@link RichGroupReduceFunction} to gain
-	 * access to other features provided by the
-	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+	 * The user can also extend the to gain access to other features provided by
+	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
 	 * </br> </br> This version of reduceGroup uses user supplied
 	 * typeinformation for serializaton. Use this only when the system is unable
-	 * to detect type information using: {@link #mapWindow(GroupReduceFunction)}
+	 * to detect type information using: {@link #mapWindow()}
 	 * 
-	 * @param reduceFunction
+	 * @param windowMapFunction
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
-	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction,
+	public <R> WindowedDataStream<R> mapWindow(WindowMapFunction<OUT, R> windowMapFunction,
 			TypeInformation<R> outType) {
 
-		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction),
-				new BasicWindowBuffer<OUT>()).mapWindow(reduceFunction, outType);
+		return discretize(WindowTransformation.MAPWINDOW.with(windowMapFunction),
+				new BasicWindowBuffer<OUT>()).mapWindow(windowMapFunction, outType);
 	}
 
-	public DataStream<OUT> flatten() {
-		return dataStream;
+	private DiscretizedStream<OUT> discretize(WindowTransformation transformation,
+			WindowBuffer<OUT> windowBuffer) {
+
+		StreamInvokable<OUT, StreamWindow<OUT>> discretizer = getDiscretizer(transformation,
+				windowBuffer, getTrigger(), getEviction(), discretizerKey);
+
+		int parallelism = getDiscretizerParallelism();
+
+		return new DiscretizedStream<OUT>(dataStream.transform("Stream Discretizer",
+				new StreamWindowTypeInfo<OUT>(getType()), discretizer).setParallelism(parallelism),
+				groupByKey, transformation);
+
 	}
 
-	protected Class<?> getClassAtPos(int pos) {
-		return dataStream.getClassAtPos(pos);
+	private int getDiscretizerParallelism() {
+		return isLocal || (discretizerKey != null) ? dataStream.environment
+				.getDegreeOfParallelism() : 1;
+	}
+
+	private StreamInvokable<OUT, StreamWindow<OUT>> getDiscretizer(
+			WindowTransformation transformation, WindowBuffer<OUT> windowBuffer,
+			TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> eviction,
+			KeySelector<OUT, ?> discretizerKey) {
+
+		if (discretizerKey == null) {
+			return new StreamDiscretizer<OUT>(trigger, eviction, windowBuffer);
+		} else if (trigger instanceof TimeTriggerPolicy
+				&& ((TimeTriggerPolicy<OUT>) trigger).timestampWrapper.isDefaultTimestamp()) {
+			return new GroupedTimeDiscretizer<OUT>(discretizerKey,
+					(TimeTriggerPolicy<OUT>) trigger, (CloneableEvictionPolicy<OUT>) eviction,
+					windowBuffer);
+		} else {
+			return new GroupedStreamDiscretizer<OUT>(discretizerKey,
+					(CloneableTriggerPolicy<OUT>) trigger, (CloneableEvictionPolicy<OUT>) eviction,
+					windowBuffer);
+		}
+
+	}
+
+	@SuppressWarnings("unchecked")
+	private WindowBuffer<OUT> getWindowBuffer(WindowTransformation transformation,
+			TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> eviction,
+			KeySelector<OUT, ?> discretizerKey) {
+
+		if (transformation == WindowTransformation.REDUCEWINDOW
+				&& eviction instanceof TumblingEvictionPolicy) {
+			if (groupByKey == null) {
+				return new TumblingPreReducer<OUT>(
+						dataStream.clean((ReduceFunction<OUT>) transformation.UDF), getType()
+								.createSerializer(getExecutionConfig()));
+			} else {
+				return new TumblingGroupedPreReducer<OUT>(
+						dataStream.clean((ReduceFunction<OUT>) transformation.UDF), groupByKey,
+						getType().createSerializer(getExecutionConfig()));
+			}
+		}
+		return new BasicWindowBuffer<OUT>();
 	}
 
 	/**
@@ -610,6 +605,10 @@ public class WindowedDataStream<OUT> {
 
 	}
 
+	protected boolean isGrouped() {
+		return groupByKey != null;
+	}
+
 	/**
 	 * Gets the output type.
 	 * 
@@ -619,21 +618,16 @@ public class WindowedDataStream<OUT> {
 		return dataStream.getType();
 	}
 
-	protected WindowedDataStream<OUT> copy() {
-		return new WindowedDataStream<OUT>(this);
-	}
-
-	protected boolean isGrouped() {
-		return groupByKey != null;
+	public ExecutionConfig getExecutionConfig() {
+		return dataStream.getExecutionConfig();
 	}
 
-	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
-		return discretize(WindowTransformation.NONE, new BasicWindowBuffer<OUT>())
-				.getDiscretizedStream();
+	protected Class<?> getClassAtPos(int pos) {
+		return dataStream.getClassAtPos(pos);
 	}
 
-	public ExecutionConfig getExecutionConfig() {
-		return dataStream.getExecutionConfig();
+	protected WindowedDataStream<OUT> copy() {
+		return new WindowedDataStream<OUT>(this);
 	}
 
 	protected static class WindowKey<R> implements KeySelector<StreamWindow<R>, Integer> {

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
index 03160c2..9af1648 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/temporaloperator/StreamCrossOperator.java
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.flink.api.common.functions.CrossFunction;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.ClosureCleaner;
 import org.apache.flink.api.java.operators.CrossOperator;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
@@ -33,16 +34,24 @@ import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
 
 public class StreamCrossOperator<I1, I2> extends
 		TemporalOperator<I1, I2, StreamCrossOperator.CrossWindow<I1, I2>> {
-	
+
 	public StreamCrossOperator(DataStream<I1> input1, DataStream<I2> input2) {
 		super(input1, input2);
 	}
 
+	protected <F> F clean(F f) {
+		if (input1.getExecutionEnvironment().getConfig().isClosureCleanerEnabled()) {
+			ClosureCleaner.clean(f, true);
+		}
+		ClosureCleaner.ensureSerializable(f);
+		return f;
+	}
+
 	@Override
 	protected CrossWindow<I1, I2> createNextWindowOperator() {
 
 		CrossWindowFunction<I1, I2, Tuple2<I1, I2>> crossWindowFunction = new CrossWindowFunction<I1, I2, Tuple2<I1, I2>>(
-				input1.clean(new CrossOperator.DefaultCrossFunction<I1, I2>()));
+				clean(new CrossOperator.DefaultCrossFunction<I1, I2>()));
 
 		return new CrossWindow<I1, I2>(this, input1.connect(input2).addGeneralWindowCombine(
 				crossWindowFunction,

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index 65dde79..8e09506 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -23,6 +23,7 @@ import java.util.Collection;
 import java.util.List;
 
 import com.esotericsoftware.kryo.Serializer;
+
 import org.apache.commons.lang3.Validate;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.io.InputFormat;

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java
new file mode 100644
index 0000000..ac2a19e
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function;
+
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.util.Collector;
+
+public abstract class RichWindowMapFunction<IN, OUT> extends AbstractRichFunction implements
+		WindowMapFunction<IN, OUT> {
+
+	private static final long serialVersionUID = 9052714915997374185L;
+
+	@Override
+	public abstract void mapWindow(Iterable<IN> values, Collector<OUT> out) throws Exception;
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java
new file mode 100644
index 0000000..273d731
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.function;
+
+import java.io.Serializable;
+
+import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.util.Collector;
+
+public interface WindowMapFunction<T, O> extends Function, Serializable {
+
+	void mapWindow(Iterable<T> values, Collector<O> out) throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
index 46d4fe9..041915f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/ParallelSourceFunction.java
@@ -1,26 +1,22 @@
 /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
  * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.function.source;
-
-import org.apache.flink.util.Collector;
-
+ */
+
+package org.apache.flink.streaming.api.function.source;
+
 public interface ParallelSourceFunction<OUT> extends SourceFunction<OUT> {
 
-	public void invoke(Collector<OUT> collector) throws Exception;
-		
-}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java
deleted file mode 100644
index 4c6e7cd..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import java.util.LinkedList;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.util.Collector;
-
-public class BasicWindowBuffer<T> implements WindowBuffer<T> {
-
-	private static final long serialVersionUID = 1L;
-	protected LinkedList<T> buffer;
-
-	public BasicWindowBuffer() {
-		this.buffer = new LinkedList<T>();
-	}
-
-	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
-		if (!buffer.isEmpty()) {
-			StreamWindow<T> currentWindow = new StreamWindow<T>();
-			currentWindow.addAll(buffer);
-			collector.collect(currentWindow);
-			return true;
-		} else {
-			return false;
-		}
-	}
-
-	public void store(T element) throws Exception {
-		buffer.add(element);
-	}
-
-	public void evict(int n) {
-		for (int i = 0; i < n; i++) {
-			try {
-				buffer.removeFirst();
-			} catch (NoSuchElementException e) {
-				// In case no more elements are in the buffer:
-				// Prevent failure and stop deleting.
-				break;
-			}
-		}
-	}
-
-	public int size() {
-		return buffer.size();
-	}
-
-	@Override
-	public BasicWindowBuffer<T> clone() {
-		return new BasicWindowBuffer<T>();
-	}
-
-	@Override
-	public String toString() {
-		return buffer.toString();
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java
deleted file mode 100644
index e071c2b..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/CompletePreAggregator.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-public interface CompletePreAggregator {
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
index ae6a2d9..58d7adb 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
@@ -23,8 +23,10 @@ import java.util.Map;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 
 public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
index 6d38ed9..5363c10 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.invokable.operator.windowing;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 
 public class GroupedTimeDiscretizer<IN> extends GroupedStreamDiscretizer<IN> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
index 6b0bcec..a14058a 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
@@ -18,11 +18,13 @@
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
 import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 
 public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
deleted file mode 100644
index 90c4e62..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.util.Collector;
-
-public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
-
-	private static final long serialVersionUID = -5150196421193988403L;
-	private static Random rnd = new Random();
-
-	public int windowID;
-	public int transformationID;
-
-	public int numberOfParts;
-
-	public StreamWindow() {
-		this(rnd.nextInt(), rnd.nextInt(), 1);
-	}
-
-	public StreamWindow(int windowID) {
-		this(windowID, rnd.nextInt(), 1);
-	}
-
-	public StreamWindow(int windowID, int transformationID, int numberOfParts) {
-		super();
-		this.windowID = windowID;
-		this.transformationID = transformationID;
-		this.numberOfParts = numberOfParts;
-	}
-
-	public StreamWindow(StreamWindow<T> window) {
-		this(window.windowID, window.transformationID, window.numberOfParts);
-		addAll(window);
-	}
-
-	public StreamWindow(StreamWindow<T> window, TypeSerializer<T> serializer) {
-		this(window.windowID, window.transformationID, window.numberOfParts);
-		for (T element : window) {
-			add(serializer.copy(element));
-		}
-	}
-
-	public List<StreamWindow<T>> partitionBy(KeySelector<T, ?> keySelector) throws Exception {
-		Map<Object, StreamWindow<T>> partitions = new HashMap<Object, StreamWindow<T>>();
-
-		for (T value : this) {
-			Object key = keySelector.getKey(value);
-			StreamWindow<T> window = partitions.get(key);
-			if (window == null) {
-				window = new StreamWindow<T>(this.windowID, this.transformationID, 0);
-				partitions.put(key, window);
-			}
-			window.add(value);
-		}
-
-		List<StreamWindow<T>> output = new ArrayList<StreamWindow<T>>();
-		int numkeys = partitions.size();
-
-		for (StreamWindow<T> window : partitions.values()) {
-			output.add(window.setNumberOfParts(numkeys));
-		}
-
-		return output;
-	}
-
-	public List<StreamWindow<T>> split(int n) {
-		int numElements = size();
-		if (n > numElements) {
-			return split(numElements);
-		} else {
-			List<StreamWindow<T>> split = new ArrayList<StreamWindow<T>>();
-			int splitSize = numElements / n;
-
-			int index = -1;
-
-			StreamWindow<T> currentSubWindow = new StreamWindow<T>(windowID, transformationID, n);
-			split.add(currentSubWindow);
-
-			for (T element : this) {
-				index++;
-				if (index == splitSize && split.size() < n) {
-					currentSubWindow = new StreamWindow<T>(windowID, transformationID, n);
-					split.add(currentSubWindow);
-					index = 0;
-				}
-				currentSubWindow.add(element);
-			}
-			return split;
-		}
-	}
-
-	public StreamWindow<T> setNumberOfParts(int n) {
-		this.numberOfParts = n;
-		return this;
-	}
-
-	public boolean compatibleWith(StreamWindow<T> otherWindow) {
-		return this.windowID == otherWindow.windowID && this.numberOfParts > 1;
-	}
-
-	public static <R> StreamWindow<R> merge(StreamWindow<R>... windows) {
-		StreamWindow<R> window = new StreamWindow<R>(windows[0]);
-		for (int i = 1; i < windows.length; i++) {
-			StreamWindow<R> next = windows[i];
-			if (window.compatibleWith(next)) {
-				window.addAll(next);
-				window.numberOfParts--;
-			} else {
-				throw new RuntimeException("Can only merge compatible windows");
-			}
-		}
-		return window;
-	}
-
-	public static <R> StreamWindow<R> merge(List<StreamWindow<R>> windows) {
-		if (windows.isEmpty()) {
-			throw new RuntimeException("Need at least one window to merge");
-		} else {
-			StreamWindow<R> window = new StreamWindow<R>(windows.get(0));
-			for (int i = 1; i < windows.size(); i++) {
-				StreamWindow<R> next = windows.get(i);
-				if (window.compatibleWith(next)) {
-					window.addAll(next);
-					window.numberOfParts--;
-				} else {
-					throw new RuntimeException("Can only merge compatible windows");
-				}
-			}
-			return window;
-		}
-	}
-
-	@Override
-	public boolean equals(Object o) {
-		return super.equals(o);
-	}
-
-	@Override
-	public void collect(T record) {
-		add(record);
-	}
-
-	@Override
-	public void close() {
-	}
-
-	@Override
-	public String toString() {
-		return super.toString();
-	}
-
-	public static <R> StreamWindow<R> fromElements(R... elements) {
-		StreamWindow<R> window = new StreamWindow<R>();
-		for (R element : elements) {
-			window.add(element);
-		}
-		return window;
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
deleted file mode 100755
index 002e440..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import java.io.IOException;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-
-public final class StreamWindowSerializer<T> extends TypeSerializer<StreamWindow<T>> {
-
-	private static final long serialVersionUID = 1L;
-
-	private final TypeSerializer<T> typeSerializer;
-	TypeSerializer<Integer> intSerializer;
-
-	public StreamWindowSerializer(TypeInformation<T> typeInfo, ExecutionConfig conf) {
-		this.typeSerializer = typeInfo.createSerializer(conf);
-		this.intSerializer = BasicTypeInfo.INT_TYPE_INFO.createSerializer(conf);
-	}
-
-	public TypeSerializer<T> getObjectSerializer() {
-		return typeSerializer;
-	}
-
-	@Override
-	public boolean isImmutableType() {
-		return false;
-	}
-
-	@Override
-	public StreamWindow<T> createInstance() {
-		return new StreamWindow<T>(0, 0, 0);
-	}
-
-	@Override
-	public StreamWindow<T> copy(StreamWindow<T> from) {
-		return new StreamWindow<T>(from, typeSerializer);
-	}
-
-	@Override
-	public StreamWindow<T> copy(StreamWindow<T> from, StreamWindow<T> reuse) {
-		reuse.clear();
-		reuse.windowID = from.windowID;
-		reuse.transformationID = from.transformationID;
-		reuse.numberOfParts = from.numberOfParts;
-
-		for (T element : from) {
-			reuse.add(typeSerializer.copy(element));
-		}
-		return reuse;
-	}
-
-	@Override
-	public int getLength() {
-		return -1;
-	}
-
-	@Override
-	public void serialize(StreamWindow<T> value, DataOutputView target) throws IOException {
-
-		intSerializer.serialize(value.windowID, target);
-		intSerializer.serialize(value.transformationID, target);
-		intSerializer.serialize(value.numberOfParts, target);
-		intSerializer.serialize(value.size(), target);
-
-		for (T element : value) {
-			typeSerializer.serialize(element, target);
-		}
-	}
-
-	@Override
-	public StreamWindow<T> deserialize(DataInputView source) throws IOException {
-		StreamWindow<T> window = createInstance();
-
-		window.windowID = intSerializer.deserialize(source);
-		window.transformationID = intSerializer.deserialize(source);
-		window.numberOfParts = intSerializer.deserialize(source);
-
-		int size = intSerializer.deserialize(source);
-
-		for (int i = 0; i < size; i++) {
-			window.add(typeSerializer.deserialize(source));
-		}
-
-		return window;
-	}
-
-	@Override
-	public StreamWindow<T> deserialize(StreamWindow<T> reuse, DataInputView source)
-			throws IOException {
-
-		StreamWindow<T> window = reuse;
-		window.clear();
-
-		window.windowID = source.readInt();
-		window.transformationID = source.readInt();
-		window.numberOfParts = source.readInt();
-
-		int size = source.readInt();
-
-		for (int i = 0; i < size; i++) {
-			window.add(typeSerializer.deserialize(source));
-		}
-
-		return window;
-	}
-
-	@Override
-	public void copy(DataInputView source, DataOutputView target) throws IOException {
-		// Needs to be implemented
-	}
-
-	@Override
-	public TypeSerializer<StreamWindow<T>> duplicate() {
-		return this;
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
deleted file mode 100644
index 0054759..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-
-public class StreamWindowTypeInfo<T> extends TypeInformation<StreamWindow<T>> {
-
-	private static final long serialVersionUID = 1L;
-	TypeInformation<T> innerType;
-
-	public StreamWindowTypeInfo(TypeInformation<T> innerType) {
-		this.innerType = innerType;
-	}
-
-	public TypeInformation<T> getInnerType() {
-		return innerType;
-	}
-
-	@Override
-	public boolean isBasicType() {
-		return innerType.isBasicType();
-	}
-
-	@Override
-	public boolean isTupleType() {
-		return innerType.isTupleType();
-	}
-
-	@Override
-	public int getArity() {
-		return innerType.getArity();
-	}
-
-	@Override
-	public Class<StreamWindow<T>> getTypeClass() {
-		// TODO Auto-generated method stub
-		return null;
-	}
-
-	@Override
-	public boolean isKeyType() {
-		return innerType.isKeyType();
-	}
-
-	@Override
-	public TypeSerializer<StreamWindow<T>> createSerializer(ExecutionConfig conf) {
-		return new StreamWindowSerializer<T>(innerType, conf);
-	}
-
-	@Override
-	public int getTotalFields() {
-		return innerType.getTotalFields();
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java
deleted file mode 100644
index f1d7ae4..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducer.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.util.Collector;
-
-public class TumblingGroupedPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
-
-	private static final long serialVersionUID = 1L;
-
-	private ReduceFunction<T> reducer;
-	private KeySelector<T, ?> keySelector;
-
-	private Map<Object, T> reducedValues;
-
-	private int numOfElements = 0;
-	private TypeSerializer<T> serializer;
-
-	public TumblingGroupedPreReducer(ReduceFunction<T> reducer, KeySelector<T, ?> keySelector,
-			TypeSerializer<T> serializer) {
-		this.reducer = reducer;
-		this.serializer = serializer;
-		this.keySelector = keySelector;
-		this.reducedValues = new HashMap<Object, T>();
-	}
-
-	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
-
-		if (!reducedValues.isEmpty()) {
-			StreamWindow<T> currentWindow = new StreamWindow<T>();
-			currentWindow.addAll(reducedValues.values());
-			collector.collect(currentWindow);
-			reducedValues.clear();
-			numOfElements = 0;
-			return true;
-		} else {
-			return false;
-		}
-
-	}
-
-	public void store(T element) throws Exception {
-		Object key = keySelector.getKey(element);
-
-		T reduced = reducedValues.get(key);
-
-		if (reduced == null) {
-			reduced = element;
-		} else {
-			reduced = reducer.reduce(serializer.copy(reduced), element);
-		}
-
-		reducedValues.put(key, reduced);
-		numOfElements++;
-	}
-
-	public void evict(int n) {
-	}
-
-	public int size() {
-		return numOfElements;
-	}
-
-	@Override
-	public TumblingGroupedPreReducer<T> clone() {
-		return new TumblingGroupedPreReducer<T>(reducer, keySelector, serializer);
-	}
-
-	@Override
-	public String toString() {
-		return reducedValues.toString();
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
deleted file mode 100644
index f1e531e..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.util.Collector;
-
-public class TumblingPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
-
-	private static final long serialVersionUID = 1L;
-
-	private ReduceFunction<T> reducer;
-
-	private T reduced;
-	private int numOfElements = 0;
-	private TypeSerializer<T> serializer;
-
-	public TumblingPreReducer(ReduceFunction<T> reducer, TypeSerializer<T> serializer) {
-		this.reducer = reducer;
-		this.serializer = serializer;
-	}
-
-	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
-		if (reduced != null) {
-			StreamWindow<T> currentWindow = new StreamWindow<T>();
-			currentWindow.add(reduced);
-			collector.collect(currentWindow);
-			reduced = null;
-			numOfElements = 0;
-			return true;
-		} else {
-			return false;
-		}
-	}
-
-	public void store(T element) throws Exception {
-		if (reduced == null) {
-			reduced = element;
-		} else {
-			reduced = reducer.reduce(serializer.copy(reduced), element);
-		}
-		numOfElements++;
-	}
-
-	public void evict(int n) {
-	}
-
-	public int size() {
-		return numOfElements;
-	}
-
-	@Override
-	public TumblingPreReducer<T> clone() {
-		return new TumblingPreReducer<T>(reducer, serializer);
-	}
-
-	@Override
-	public String toString() {
-		return reduced.toString();
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java
deleted file mode 100644
index ef8fc2b..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import java.io.Serializable;
-
-import org.apache.flink.util.Collector;
-
-public interface WindowBuffer<T> extends Serializable {
-
-	public void store(T element) throws Exception;
-
-	public void evict(int n);
-
-	public boolean emitWindow(Collector<StreamWindow<T>> collector);
-
-	public int size();
-	
-	public WindowBuffer<T> clone();
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
index 5f5e7d2..7eaea58 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
@@ -18,6 +18,7 @@
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 
 public class WindowFlattener<T> extends ChainableInvokable<StreamWindow<T>, T> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
index 23aaf32..de93fab 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
@@ -17,29 +17,29 @@
 
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
-import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 
 public class WindowMapper<IN, OUT> extends MapInvokable<StreamWindow<IN>, StreamWindow<OUT>> {
 
 	private static final long serialVersionUID = 1L;
 
-	GroupReduceFunction<IN, OUT> reducer;
+	WindowMapFunction<IN, OUT> mapper;
 
-	public WindowMapper(GroupReduceFunction<IN, OUT> reducer) {
-		super(new WindowMapfunction<IN, OUT>(reducer));
-		this.reducer = reducer;
+	public WindowMapper(WindowMapFunction<IN, OUT> mapper) {
+		super(new WindowMap<IN, OUT>(mapper));
+		this.mapper = mapper;
 	}
 
-	private static class WindowMapfunction<T, R> implements
-			MapFunction<StreamWindow<T>, StreamWindow<R>> {
+	private static class WindowMap<T, R> implements MapFunction<StreamWindow<T>, StreamWindow<R>> {
 
 		private static final long serialVersionUID = 1L;
-		GroupReduceFunction<T, R> reducer;
+		WindowMapFunction<T, R> mapper;
 
-		public WindowMapfunction(GroupReduceFunction<T, R> reducer) {
-			this.reducer = reducer;
+		public WindowMap(WindowMapFunction<T, R> mapper) {
+			this.mapper = mapper;
 		}
 
 		@Override
@@ -47,7 +47,7 @@ public class WindowMapper<IN, OUT> extends MapInvokable<StreamWindow<IN>, Stream
 			StreamWindow<R> outputWindow = new StreamWindow<R>(window.windowID);
 			outputWindow.numberOfParts = window.numberOfParts;
 
-			reducer.reduce(window, outputWindow);
+			mapper.mapWindow(window, outputWindow);
 
 			return outputWindow;
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
index 1766b0b..8601d06 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
@@ -21,6 +21,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 
 public class WindowMerger<T> extends ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
index e10692b..ea4451e 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
@@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 
 public class WindowPartitioner<T> extends
 		ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
index b4f965f..3a4bb69 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.invokable.operator.windowing;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 
 public class WindowReducer<IN> extends MapInvokable<StreamWindow<IN>, StreamWindow<IN>> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
new file mode 100644
index 0000000..988058c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.util.Collector;
+
+public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
+
+	private static final long serialVersionUID = -5150196421193988403L;
+	private static Random rnd = new Random();
+
+	public int windowID;
+
+	public int numberOfParts;
+
+	public StreamWindow() {
+		this(rnd.nextInt(), 1);
+	}
+
+	public StreamWindow(int windowID) {
+		this(windowID, 1);
+	}
+
+	public StreamWindow(int windowID, int numberOfParts) {
+		super();
+		this.windowID = windowID;
+		this.numberOfParts = numberOfParts;
+	}
+
+	public StreamWindow(StreamWindow<T> window) {
+		this(window.windowID, window.numberOfParts);
+		addAll(window);
+	}
+
+	public StreamWindow(StreamWindow<T> window, TypeSerializer<T> serializer) {
+		this(window.windowID, window.numberOfParts);
+		for (T element : window) {
+			add(serializer.copy(element));
+		}
+	}
+
+	public List<StreamWindow<T>> partitionBy(KeySelector<T, ?> keySelector) throws Exception {
+		Map<Object, StreamWindow<T>> partitions = new HashMap<Object, StreamWindow<T>>();
+
+		for (T value : this) {
+			Object key = keySelector.getKey(value);
+			StreamWindow<T> window = partitions.get(key);
+			if (window == null) {
+				window = new StreamWindow<T>(this.windowID, 0);
+				partitions.put(key, window);
+			}
+			window.add(value);
+		}
+
+		List<StreamWindow<T>> output = new ArrayList<StreamWindow<T>>();
+		int numkeys = partitions.size();
+
+		for (StreamWindow<T> window : partitions.values()) {
+			output.add(window.setNumberOfParts(numkeys));
+		}
+
+		return output;
+	}
+
+	public List<StreamWindow<T>> split(int n) {
+		int numElements = size();
+		if (n > numElements) {
+			return split(numElements);
+		} else {
+			List<StreamWindow<T>> split = new ArrayList<StreamWindow<T>>();
+			int splitSize = numElements / n;
+
+			int index = -1;
+
+			StreamWindow<T> currentSubWindow = new StreamWindow<T>(windowID, n);
+			split.add(currentSubWindow);
+
+			for (T element : this) {
+				index++;
+				if (index == splitSize && split.size() < n) {
+					currentSubWindow = new StreamWindow<T>(windowID, n);
+					split.add(currentSubWindow);
+					index = 0;
+				}
+				currentSubWindow.add(element);
+			}
+			return split;
+		}
+	}
+
+	public StreamWindow<T> setNumberOfParts(int n) {
+		this.numberOfParts = n;
+		return this;
+	}
+
+	public boolean compatibleWith(StreamWindow<T> otherWindow) {
+		return this.windowID == otherWindow.windowID && this.numberOfParts > 1;
+	}
+
+	public static <R> StreamWindow<R> merge(StreamWindow<R>... windows) {
+		StreamWindow<R> window = new StreamWindow<R>(windows[0]);
+		for (int i = 1; i < windows.length; i++) {
+			StreamWindow<R> next = windows[i];
+			if (window.compatibleWith(next)) {
+				window.addAll(next);
+				window.numberOfParts--;
+			} else {
+				throw new RuntimeException("Can only merge compatible windows");
+			}
+		}
+		return window;
+	}
+
+	public static <R> StreamWindow<R> merge(List<StreamWindow<R>> windows) {
+		if (windows.isEmpty()) {
+			throw new RuntimeException("Need at least one window to merge");
+		} else {
+			StreamWindow<R> window = new StreamWindow<R>(windows.get(0));
+			for (int i = 1; i < windows.size(); i++) {
+				StreamWindow<R> next = windows.get(i);
+				if (window.compatibleWith(next)) {
+					window.addAll(next);
+					window.numberOfParts--;
+				} else {
+					throw new RuntimeException("Can only merge compatible windows");
+				}
+			}
+			return window;
+		}
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		return super.equals(o);
+	}
+
+	@Override
+	public void collect(T record) {
+		add(record);
+	}
+
+	@Override
+	public void close() {
+	}
+
+	@Override
+	public String toString() {
+		return super.toString();
+	}
+
+	public static <R> StreamWindow<R> fromElements(R... elements) {
+		StreamWindow<R> window = new StreamWindow<R>();
+		for (R element : elements) {
+			window.add(element);
+		}
+		return window;
+	}
+}


[20/20] flink git commit: [docs] [streaming] Documentation updates for stream windowing

Posted by mb...@apache.org.
[docs] [streaming] Documentation updates for stream windowing

This closes #395


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e30a2ea2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e30a2ea2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e30a2ea2

Branch: refs/heads/master
Commit: e30a2ea2fd5eb9f65e89e44e2f04458d1502e4d2
Parents: 3d10a23
Author: mbalassi <mb...@apache.org>
Authored: Mon Feb 16 15:22:56 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 15:22:56 2015 +0100

----------------------------------------------------------------------
 docs/streaming_guide.md | 78 +++++++++++++++++++-------------------------
 1 file changed, 33 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/e30a2ea2/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index e409b49..3d6b75e 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -34,7 +34,7 @@ Flink Streaming is an extension of the core Flink API for high-throughput, low-l
 Flink Streaming API
 -----------
 
-The Streaming API is currently part of the *addons* Maven project. All relevant classes are located in the *org.apache.flink.streaming* package.
+The Streaming API is currently part of the *flink-staging* Maven project. All relevant classes are located in the *org.apache.flink.streaming* package.
 
 Add the following dependency to your `pom.xml` to use the Flink Streaming.
 
@@ -42,7 +42,7 @@ Add the following dependency to your `pom.xml` to use the Flink Streaming.
 <dependency>
     <groupId>org.apache.flink</groupId>
     <artifactId>flink-streaming-core</artifactId>
-    <version>{{site.FLINK_VERSION_STABLE}}</version>
+    <version>{{site.FLINK_VERSION_SHORT}}</version>
 </dependency>
 ~~~
 
@@ -284,7 +284,7 @@ The Flink Streaming API supports different types of pre-defined aggregation oper
 
 Types of aggregations: `sum(field)`, `min(field)`, `max(field)`, `minBy(field, first)`, `maxBy(field, first)`
 
-With `sum`, `min`, and `max` for every incoming tuple the selected field is replaced with the current aggregated value. Fields can be selected using either field positions or field expressions (similarly to grouping).  
+With `sum`, `min`, and `max` for every incoming tuple the selected field is replaced with the current aggregated value. Fields can be selected using either field positions or field expressions (similarly to grouping).
 
 With `minBy` and `maxBy` the output of the operator is the element with the current minimal or maximal value at the given field. If more components share the minimum or maximum value, the user can decide if the operator should return the first or last element. This can be set by the `first` boolean parameter.
 
@@ -300,14 +300,15 @@ The user can control the size (eviction) of the windows and the frequency of red
  * `dataStream.window(…).every(…).mapWindow(…).flatten()`
  * `dataStream.window(…).every(…).groupBy(…).aggregate(…).getDiscretizedStream()`
 
-The core abstraction of the Windowing semantics is the `WindowedDataStream` and the `StreamWindow`. The WindowedDataStream which is created when we call the `.window(…)` method of the DataStream, represents the windowed discretisation of the underlying stream. The user can think about it simply as a `DataStream<StreamWindow<T>>` where additional API functions are supplied to provide efficient transformations of individual windows. 
+The core abstraction of the Windowing semantics is the `WindowedDataStream` and the `StreamWindow`. The `WindowedDataStream` is created when we call the `.window(…)` method of the DataStream and represents the windowed discretisation of the underlying stream. The user can think about it simply as a `DataStream<StreamWindow<T>>` where additional API functions are supplied to provide efficient transformations of individual windows. 
 
-The result of a window transformation is again a `WindowedDataStream` which can also be used to further transform the resulting windows. In this sense, window transformations define mapping from `StreamWindow -> StreamWindow’`.
+The result of a window transformation is again a `WindowedDataStream` which can also be used to further transform the resulting windows. In this sense, window transformations define mapping from stream windows to stream windows.
 
-The user have different ways of working further with a result of a window operation:
- * `windowedDataStream.flatten()` - Which streams out the results element wise and returns a `DataStream<T>` where T is the type of the underlying windowed stream
- * `windowedDataStream.getDiscretizedStream()` - Which returns a `DataStream<StreamWindow<T>>` for applying some advanced logic on the stream windows itself
- * Calling any window transformation further transforms the windows
+The user has different ways of using the a result of a window operation:
+
+ * `windowedDataStream.flatten()` - streams the results element wise and returns a `DataStream<T>` where T is the type of the underlying windowed stream
+ * `windowedDataStream.getDiscretizedStream()` - returns a `DataStream<StreamWindow<T>>` for applying some advanced logic on the stream windows itself
+ * Calling any window transformation further transforms the windows, while preserving the windowing logic
 
 The next example would create windows that hold elements of the last 5 seconds, and the user defined transformation would be executed on the windows every second (sliding the window by 1 second):
 
@@ -315,7 +316,7 @@ The next example would create windows that hold elements of the last 5 seconds,
 dataStream.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(1, TimeUnit.SECONDS))
 ~~~
 
-This approach is often referred to as policy based windowing. Different policies (count, time, etc.) can be mixed as well; for example to downsample our stream, a window that takes the latest 100 elements of the stream every minute is created as follows:
+This approach is often referred to as policy based windowing. Different policies (count, time, etc.) can be mixed as well, for example to downsample our stream, a window that takes the latest 100 elements of the stream every minute is created as follows:
 
 ~~~java
 dataStream.window(Count.of(100)).every(Time.of(1, TimeUnit.MINUTES))
@@ -329,7 +330,7 @@ Several predefined policies are provided in the API, including delta-based, coun
  * `Count.of(…)`
  * `Delta.of(…)`
 
-For detailed description of these policies please refer to the javadocs.
+For detailed description of these policies please refer to the [Javadocs](http://flink.apache.org/docs/latest/api/java/).
 
 #### Policy based windowing
 The policy based windowing is a highly flexible way to specify stream discretisation also called windowing semantics. Two types of policies are used for such a specification:
@@ -351,16 +352,16 @@ By default most triggers can only trigger when a new element arrives. This might
 Time-based trigger and eviction policies can work with user defined `TimeStamp` implementations, these policies already cover most use cases.
  
 #### Reduce on windowed data streams
-The `WindowedDataStream<T>.reduceWindow(ReduceFunction<T>)` transformation calls the user-defined `ReduceFunction` at every trigger on the records currently in the window. The user can also use the different pre-implemented streaming aggregations: `sum, min, max, minBy, maxBy`
+The `WindowedDataStream<T>.reduceWindow(ReduceFunction<T>)` transformation calls the user-defined `ReduceFunction` at every trigger on the records currently in the window. The user can also use the different pre-implemented streaming aggregations such as `sum, min, max, minBy` and `maxBy`.
 
-A window reduce that sums the elements in the last minute with 10 seconds slide interval:
+The following is an example for a window reduce that sums the elements in the last minute with 10 seconds slide interval:
 
 ~~~java
 dataStream.window(Time.of(1, TimeUnit.MINUTES)).every(Time.of(10,TimeUnit.SECONDS)).sum(field);
 ~~~
 
 #### Map on windowed data streams
-The `WindowedDataStream<T>.mapWindow(WindowMapFunction<T,O>)` transformation calls  `WindowMapFunction.mapWindow(…)` for each `StreamWindow` in the discretised stream providing access to all elements in the window through the iterable interface. At each function call the output `StreamWindow<O>` will consist of all the elements collected to the collector. This allows a straightforward way of mapping one stream window to another.
+The `WindowedDataStream<T>.mapWindow(WindowMapFunction<T,O>)` transformation calls  `mapWindow(…)` for each `StreamWindow` in the discretised stream providing access to all elements in the window through the iterable interface. At each function call the output `StreamWindow<O>` will consist of all the elements collected to the collector. This allows a straightforward way of mapping one stream window to another.
 
 ~~~java
 windowedDataStream.mapWindow(windowMapFunction)
@@ -379,9 +380,7 @@ To get the maximal value for each key on the last 100 elements (global) we use t
 dataStream.window(Count.of(100)).every(…).groupBy(groupingField).max(field);
 ~~~
 
-Using this approach we took the last 100 elements, divided it into groups by key then applied the aggregation.
-
-To create fixed size windows for every key we need to reverse the order of the groupBy call. So to take the max for the last 100 elements in Each group:
+Using this approach we took the last 100 elements, divided it into groups by key then applied the aggregation. To create fixed size windows for every key we need to reverse the order of the groupBy call. So to take the max for the last 100 elements in Each group:
 
 ~~~java
 dataStream.groupBy(groupingField).window(Count.of(100)).every(…).max(field);
@@ -393,31 +392,20 @@ This will create separate windows for different keys and apply the trigger and e
 Using the `WindowedDataStream` abstraction we can apply several transformations one after another on the discretised streams without having to re-discretise it:
 
 ~~~java
-dataStream.window(Count.of(1000)).groupBy(firstKey).mapWindow(…).groupBy(secondKey).reduceWindow(…).flatten()
+dataStream.window(Count.of(1000)).groupBy(firstKey).mapWindow(…)
+    .groupBy(secondKey).reduceWindow(…).flatten()
 ~~~
 
 The above call would create global windows of 1000 elements group it by the first key and then apply a mapWindow transformation. The resulting windowed stream will then be grouped by the second key and further reduced. The results of the reduce transformation are then flattened.
 
-Notice here that we only defined the window size once at the beginning of the transformation. This means that anything that happens afterwards (`.groupBy(firstKey).mapWindow(…).groupBy(secondKey).reduceWindow(…)`) happens inside the 1000 element windows. Of course the mapWindow might reduce the number of elements but the idea is that each transformation still corresponds to the same 1000 elements in the original stream.
+Notice here that we only defined the window size once at the beginning of the transformation. This means that anything that happens afterwards (`.groupBy(firstKey).mapWindow(…).groupBy(secondKey).reduceWindow(…)`) happens inside the 1000 element windows. Of course the mapWindow might reduce the number of elements but the key idea is that each transformation still corresponds to the same 1000 elements in the original stream.
 
 #### Global vs local discretisation
-By default all window discretisation calls (`dataStream.window(…)`) define global windows meaning that a global window of count 100 will contain the last 100 elements arrived at the discretisation operator in order. In most cases (except for Time) this means that the operator doing the actual discretisation needs to have a degree of parallelism of 1 to be able to correctly execute the discretisation logic. 
-
-Sometimes it is enough to create local discretisations, which allows the discretiser to run in parallel and apply the given discretisation logic at every discretiser instance. To allow local discretisation use the `.local()` method of the windowed data stream.
-
-Example:
-
-~~~java
-dataStream.window(Count.of(100)).maxBy(field);
-~~~
-
-This would create global windows of 100 elements (Count discretises with parallelism of 1) and return the record with the max value by the selected field, while
+By default all window discretisation calls (`dataStream.window(…)`) define global windows meaning that a global window of count 100 will contain the last 100 elements arrived at the discretisation operator in order. In most cases (except for Time) this means that the operator doing the actual discretisation needs to have a degree of parallelism of 1 to be able to correctly execute the discretisation logic.
 
-~~~java
-dataStream.window(Count.of(100)).local().maxBy(field);
-~~~
+Sometimes it is sufficient to create local discretisations, which allows the discretiser to run in parallel and apply the given discretisation logic at every discretiser instance. To allow local discretisation use the `local()` method of the windowed data stream.
 
-this would create several count discretisers (as defined by the environment parallelism) and compute the max values accordingly.
+For example `dataStream.window(Count.of(100)).maxBy(field)` would create global windows of 100 elements (Count discretises with parallelism of 1) and return the record with the max value by the selected field, alternatively the `dataStream.window(Count.of(100)).local().maxBy(field)` would create several count discretisers (as defined by the environment parallelism) and compute the max values accordingly.
 
 
 ### Temporal database style operators
@@ -432,9 +420,9 @@ The following code shows a default Join transformation using field position keys
 
 ~~~java
 dataStream1.join(dataStream2)
-		.onWindow(windowing_params)
-		.where(key_in_first)
-		.equalTo(key_in_second);
+    .onWindow(windowing_params)
+    .where(key_in_first)
+    .equalTo(key_in_second);
 ~~~
 
 The Cross transformation combines two DataStreams into one DataStreams. It builds all pairwise combinations of the elements of both input DataStreams in the current window, i.e., it builds a temporal Cartesian product.
@@ -527,7 +515,7 @@ For example to split even and odd numbers:
 
 ~~~java
 @Override
-Iterable<String> select(Integer value) {	
+Iterable<String> select(Integer value) {
 
     List<String> outputs = new ArrayList<String>();
 
@@ -622,7 +610,7 @@ For a detailed Java 8 Guide please refer to the [Java 8 Programming Guide](java8
 
 ~~~java
 SplitDataStream<Integer> split = someDataStream
-					.split(x -> Arrays.asList(String.valueOf(x % 2)));
+    .split(x -> Arrays.asList(String.valueOf(x % 2)));
 ~~~
 
 Operator Settings
@@ -646,7 +634,7 @@ env.setBufferTimeout(timeoutMillis);
 env.genereateSequence(1,10).map(new MyMapper()).setBufferTimeout(timeoutMillis);
 ~~~
 
-To maximise the throughput the user can call `.setBufferTimeout(-1)` which will remove the timeout and buffers will only be flushed when they are full.
+To maximise the throughput the user can call `setBufferTimeout(-1)` which will remove the timeout and buffers will only be flushed when they are full.
 To minimise latency, set the timeout to a value close to 0 (fro example 5 or 10 ms). Theoretically a buffer timeout of 0 will cause all outputs to be flushed when produced, but this setting should be avoided because it can cause severe performance degradation.
 
 
@@ -673,7 +661,7 @@ This connector provides access to data streams from [Apache Kafka](https://kafka
 #### Kafka Source
 A class providing an interface for receiving data from Kafka.
 
-The followings have to be provided for the `KafkaSource(..)` constructor in order:
+The followings have to be provided for the `KafkaSource(…)` constructor in order:
 
 1. The hostname
 2. The group name
@@ -895,7 +883,7 @@ After installing Docker an image can be pulled for each connector. Containers ca
 For the easiest set up create a jar with all the dependencies of the *flink-streaming-connectors* project.
 
 ~~~batch
-cd /PATH/TO/GIT/incubator-flink/flink-addons/flink-streaming-connectors
+cd /PATH/TO/GIT/flink/flink-staging/flink-streaming-connectors
 mvn assembly:assembly
 ~~~batch
 
@@ -1012,14 +1000,14 @@ Now a terminal started running from the image with all the necessary configurati
 
 To have the latest version of Flink type:
 ~~~batch
-cd /git/incubator-flink/
+cd /git/flink/
 git pull
 ~~~
 
 Then build the code with:
 
 ~~~batch
-cd /git/incubator-flink/flink-addons/flink-streaming/flink-streaming-connectors/
+cd /git/flink/flink-staging/flink-streaming/flink-streaming-connectors/
 mvn install -DskipTests
 ~~~
 
@@ -1047,4 +1035,4 @@ In the example there are to connectors. One that sends messages to Flume and one
 <DATE> INFO flume.FlumeTopology: String: <five> arrived from Flume
 ~~~
 
-[Back to top](#top)
\ No newline at end of file
+[Back to top](#top)


[18/20] flink git commit: [FLINK-1539] [streaming] Remove calls to uninitalized runtimecontexts

Posted by mb...@apache.org.
[FLINK-1539] [streaming] Remove calls to uninitalized runtimecontexts


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/44702075
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/44702075
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/44702075

Branch: refs/heads/master
Commit: 4470207501ff318bd4d59bad58a5f3fc4ccbfa6f
Parents: bb5dc7e
Author: Gyula Fora <gy...@apache.org>
Authored: Fri Feb 13 19:57:55 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../streaming/api/datastream/DataStream.java    | 11 ++++----
 .../api/datastream/WindowedDataStream.java      | 12 ++++-----
 .../aggregation/ComparableAggregator.java       | 14 +++++-----
 .../api/function/aggregation/SumAggregator.java | 10 +++++---
 .../api/function/source/FileSourceFunction.java | 15 +++--------
 .../api/invokable/StreamInvokable.java          |  5 ++--
 .../invokable/operator/ProjectInvokable.java    |  4 ++-
 .../api/invokable/operator/co/CoInvokable.java  |  3 +--
 .../api/streamvertex/CoStreamVertex.java        |  2 +-
 .../api/streamvertex/StreamTaskContext.java     |  5 +++-
 .../api/streamvertex/StreamVertex.java          |  2 +-
 .../windowing/windowbuffer/WindowBuffer.java    |  4 +--
 .../flink/streaming/util/MockCoContext.java     |  7 ++++-
 .../flink/streaming/util/MockContext.java       |  7 ++++-
 .../src/test/resources/log4j-test.properties    | 10 +++++++-
 .../src/test/resources/log4j.properties         | 27 ++++++++++++++++++++
 16 files changed, 92 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index e766626..71a97f8 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -636,7 +636,8 @@ public class DataStream<OUT> {
 	 * @return The transformed DataStream.
 	 */
 	public SingleOutputStreamOperator<OUT, ?> sum(String field) {
-		return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType()));
+		return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType(),
+				getExecutionConfig()));
 	}
 
 	/**
@@ -667,7 +668,7 @@ public class DataStream<OUT> {
 	 */
 	public SingleOutputStreamOperator<OUT, ?> min(String field) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MIN,
-				false));
+				false, getExecutionConfig()));
 	}
 
 	/**
@@ -698,7 +699,7 @@ public class DataStream<OUT> {
 	 */
 	public SingleOutputStreamOperator<OUT, ?> max(String field) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MAX,
-				false));
+				false, getExecutionConfig()));
 	}
 
 	/**
@@ -718,7 +719,7 @@ public class DataStream<OUT> {
 	 */
 	public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(),
-				AggregationType.MINBY, first));
+				AggregationType.MINBY, first, getExecutionConfig()));
 	}
 
 	/**
@@ -738,7 +739,7 @@ public class DataStream<OUT> {
 	 */
 	public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(),
-				AggregationType.MAXBY, first));
+				AggregationType.MAXBY, first, getExecutionConfig()));
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 3ff5859..4da12ac 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -381,7 +381,8 @@ public class WindowedDataStream<OUT> {
 	 * @return The transformed DataStream.
 	 */
 	public WindowedDataStream<OUT> sum(String field) {
-		return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType()));
+		return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType(),
+				getExecutionConfig()));
 	}
 
 	/**
@@ -411,7 +412,7 @@ public class WindowedDataStream<OUT> {
 	 */
 	public WindowedDataStream<OUT> min(String field) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MIN,
-				false));
+				false, getExecutionConfig()));
 	}
 
 	/**
@@ -475,7 +476,7 @@ public class WindowedDataStream<OUT> {
 	 */
 	public WindowedDataStream<OUT> minBy(String field, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(),
-				AggregationType.MINBY, first));
+				AggregationType.MINBY, first, getExecutionConfig()));
 	}
 
 	/**
@@ -505,7 +506,7 @@ public class WindowedDataStream<OUT> {
 	 */
 	public WindowedDataStream<OUT> max(String field) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MAX,
-				false));
+				false, getExecutionConfig()));
 	}
 
 	/**
@@ -569,11 +570,10 @@ public class WindowedDataStream<OUT> {
 	 */
 	public WindowedDataStream<OUT> maxBy(String field, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(),
-				AggregationType.MAXBY, first));
+				AggregationType.MAXBY, first, getExecutionConfig()));
 	}
 
 	private WindowedDataStream<OUT> aggregate(AggregationFunction<OUT> aggregator) {
-
 		return reduceWindow(aggregator);
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
index 7f7cf0b..66da931 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/ComparableAggregator.java
@@ -21,6 +21,7 @@ import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.util.List;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -65,9 +66,10 @@ public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
 	}
 
 	public static <R> AggregationFunction<R> getAggregator(String field,
-			TypeInformation<R> typeInfo, AggregationType aggregationType, boolean first) {
+			TypeInformation<R> typeInfo, AggregationType aggregationType, boolean first,
+			ExecutionConfig config) {
 
-		return new PojoComparableAggregator<R>(field, typeInfo, aggregationType, first);
+		return new PojoComparableAggregator<R>(field, typeInfo, aggregationType, first, config);
 	}
 
 	private static class TupleComparableAggregator<T> extends ComparableAggregator<T> {
@@ -177,7 +179,7 @@ public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
 		PojoComparator<T> pojoComparator;
 
 		public PojoComparableAggregator(String field, TypeInformation<?> typeInfo,
-				AggregationType aggregationType, boolean first) {
+				AggregationType aggregationType, boolean first, ExecutionConfig config) {
 			super(0, aggregationType, first);
 			if (!(typeInfo instanceof CompositeType<?>)) {
 				throw new IllegalArgumentException(
@@ -193,7 +195,7 @@ public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
 
 			if (cType instanceof PojoTypeInfo) {
 				pojoComparator = (PojoComparator<T>) cType.createComparator(
-						new int[] { logicalKeyPosition }, new boolean[] { false }, 0, getRuntimeContext().getExecutionConfig());
+						new int[] { logicalKeyPosition }, new boolean[] { false }, 0, config);
 			} else {
 				throw new IllegalArgumentException(
 						"Key expressions are only supported on POJO types. "
@@ -225,8 +227,8 @@ public abstract class ComparableAggregator<T> extends AggregationFunction<T> {
 			} else {
 				if (c == 1) {
 					keyFields[0].set(value2, field1);
-				} 
-				
+				}
+
 				return value2;
 			}
 		}

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
index 74e4597..20d4450 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/aggregation/SumAggregator.java
@@ -21,6 +21,7 @@ import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.util.List;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
@@ -47,9 +48,10 @@ public abstract class SumAggregator {
 
 	}
 
-	public static <T> ReduceFunction<T> getSumFunction(String field, TypeInformation<T> typeInfo) {
+	public static <T> ReduceFunction<T> getSumFunction(String field, TypeInformation<T> typeInfo,
+			ExecutionConfig config) {
 
-		return new PojoSumAggregator<T>(field, typeInfo);
+		return new PojoSumAggregator<T>(field, typeInfo, config);
 	}
 
 	private static class TupleSumAggregator<T> extends AggregationFunction<T> {
@@ -126,7 +128,7 @@ public abstract class SumAggregator {
 		SumFunction adder;
 		PojoComparator<T> comparator;
 
-		public PojoSumAggregator(String field, TypeInformation<?> type) {
+		public PojoSumAggregator(String field, TypeInformation<?> type, ExecutionConfig config) {
 			super(0);
 			if (!(type instanceof CompositeType<?>)) {
 				throw new IllegalArgumentException(
@@ -146,7 +148,7 @@ public abstract class SumAggregator {
 
 			if (cType instanceof PojoTypeInfo) {
 				comparator = (PojoComparator<T>) cType.createComparator(
-						new int[] { logicalKeyPosition }, new boolean[] { false }, 0, getRuntimeContext().getExecutionConfig());
+						new int[] { logicalKeyPosition }, new boolean[] { false }, 0, config);
 			} else {
 				throw new IllegalArgumentException(
 						"Key expressions are only supported on POJO types. "

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
index 6d1441a..20f5f56 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/source/FileSourceFunction.java
@@ -23,8 +23,6 @@ import java.util.NoSuchElementException;
 import org.apache.flink.api.common.io.InputFormat;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.api.java.typeutils.runtime.RuntimeSerializerFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.InputSplit;
 import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
@@ -38,17 +36,11 @@ public class FileSourceFunction extends RichSourceFunction<String> {
 
 	private InputFormat<String, ?> inputFormat;
 
-	private TypeSerializerFactory<String> serializerFactory;
+	private TypeInformation<String> typeInfo;
 
 	public FileSourceFunction(InputFormat<String, ?> format, TypeInformation<String> typeInfo) {
 		this.inputFormat = format;
-		this.serializerFactory = createSerializer(typeInfo);
-	}
-
-	private TypeSerializerFactory<String> createSerializer(TypeInformation<String> typeInfo) {
-		TypeSerializer<String> serializer = typeInfo.createSerializer(getRuntimeContext().getExecutionConfig());
-
-		return new RuntimeSerializerFactory<String>(serializer, typeInfo.getTypeClass());
+		this.typeInfo = typeInfo;
 	}
 
 	@Override
@@ -60,7 +52,8 @@ public class FileSourceFunction extends RichSourceFunction<String> {
 
 	@Override
 	public void invoke(Collector<String> collector) throws Exception {
-		final TypeSerializer<String> serializer = serializerFactory.getSerializer();
+		final TypeSerializer<String> serializer = typeInfo.createSerializer(getRuntimeContext()
+				.getExecutionConfig());
 		final Iterator<InputSplit> splitIterator = getInputSplits();
 		@SuppressWarnings("unchecked")
 		final InputFormat<String, InputSplit> format = (InputFormat<String, InputSplit>) this.inputFormat;

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
index 7feeac8..733edc7 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
@@ -73,9 +73,8 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 	 * 
 	 * @param taskContext
 	 *            StreamTaskContext representing the vertex
-	 * @param executionConfig
 	 */
-	public void setup(StreamTaskContext<OUT> taskContext, ExecutionConfig executionConfig) {
+	public void setup(StreamTaskContext<OUT> taskContext) {
 		this.collector = taskContext.getOutputCollector();
 		this.recordIterator = taskContext.getIndexedInput(0);
 		this.inSerializer = taskContext.getInputSerializer(0);
@@ -84,7 +83,7 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 			this.objectSerializer = inSerializer.getObjectSerializer();
 		}
 		this.taskContext = taskContext;
-		this.executionConfig = executionConfig;
+		this.executionConfig = taskContext.getExecutionConfig();
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
index 69c7cee..31689c7 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/ProjectInvokable.java
@@ -28,6 +28,7 @@ public class ProjectInvokable<IN, OUT extends Tuple> extends StreamInvokable<IN,
 
 	transient OUT outTuple;
 	TypeSerializer<OUT> outTypeSerializer;
+	TypeInformation<OUT> outTypeInformation;
 	int[] fields;
 	int numFields;
 
@@ -35,7 +36,7 @@ public class ProjectInvokable<IN, OUT extends Tuple> extends StreamInvokable<IN,
 		super(null);
 		this.fields = fields;
 		this.numFields = this.fields.length;
-		this.outTypeSerializer = outTypeInformation.createSerializer(executionConfig);
+		this.outTypeInformation = outTypeInformation;
 	}
 
 	@Override
@@ -56,6 +57,7 @@ public class ProjectInvokable<IN, OUT extends Tuple> extends StreamInvokable<IN,
 	@Override
 	public void open(Configuration config) throws Exception {
 		super.open(config);
+		this.outTypeSerializer = outTypeInformation.createSerializer(executionConfig);
 		outTuple = outTypeSerializer.createInstance();
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
index 9f98db3..604873e 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/co/CoInvokable.java
@@ -17,7 +17,6 @@
 
 package org.apache.flink.streaming.api.invokable.operator.co;
 
-import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
@@ -47,7 +46,7 @@ public abstract class CoInvokable<IN1, IN2, OUT> extends StreamInvokable<IN1, OU
 	protected TypeSerializer<IN2> serializer2;
 
 	@Override
-	public void setup(StreamTaskContext<OUT> taskContext, ExecutionConfig executionConfig) {
+	public void setup(StreamTaskContext<OUT> taskContext) {
 		this.collector = taskContext.getOutputCollector();
 
 		this.recordIterator = taskContext.getCoReader();

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
index 7b6e75e..de4660a 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/CoStreamVertex.java
@@ -69,7 +69,7 @@ public class CoStreamVertex<IN1, IN2, OUT> extends StreamVertex<IN1, OUT> {
 	@Override
 	protected void setInvokable() {
 		userInvokable = configuration.getUserInvokable(userClassLoader);
-		userInvokable.setup(this, getExecutionConfig());
+		userInvokable.setup(this);
 	}
 
 	protected void setConfigInputs() throws StreamVertexException {

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
index 665decd..1c904ca 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamTaskContext.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.streaming.api.streamvertex;
 
+import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.streaming.api.StreamConfig;
 import org.apache.flink.streaming.api.streamrecord.StreamRecordSerializer;
 import org.apache.flink.streaming.io.CoReaderIterator;
@@ -32,7 +33,7 @@ public interface StreamTaskContext<OUT> {
 	ClassLoader getUserCodeClassLoader();
 
 	<X> MutableObjectIterator<X> getInput(int index);
-	
+
 	<X> IndexedReaderIterator<X> getIndexedInput(int index);
 
 	<X> StreamRecordSerializer<X> getInputSerializer(int index);
@@ -40,4 +41,6 @@ public interface StreamTaskContext<OUT> {
 	Collector<OUT> getOutputCollector();
 
 	<X, Y> CoReaderIterator<X, Y> getCoReader();
+
+	ExecutionConfig getExecutionConfig();
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
index 91ffec1..5033357 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/streamvertex/StreamVertex.java
@@ -98,7 +98,7 @@ public class StreamVertex<IN, OUT> extends AbstractInvokable implements StreamTa
 
 	protected void setInvokable() {
 		userInvokable = configuration.getUserInvokable(userClassLoader);
-		userInvokable.setup(this, getExecutionConfig());
+		userInvokable.setup(this);
 	}
 
 	public String getName() {

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
index 59abcd6..1a45194 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.util.Collector;
 
-public interface WindowBuffer<T> extends Serializable {
+public interface WindowBuffer<T> extends Serializable, Cloneable {
 
 	public void store(T element) throws Exception;
 
@@ -31,7 +31,7 @@ public interface WindowBuffer<T> extends Serializable {
 	public boolean emitWindow(Collector<StreamWindow<T>> collector);
 
 	public int size();
-	
+
 	public WindowBuffer<T> clone();
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
index a0d08f1..98a6a8d 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
@@ -156,7 +156,7 @@ public class MockCoContext<IN1, IN2, OUT> implements StreamTaskContext<OUT> {
 	public static <IN1, IN2, OUT> List<OUT> createAndExecute(CoInvokable<IN1, IN2, OUT> invokable,
 			List<IN1> input1, List<IN2> input2) {
 		MockCoContext<IN1, IN2, OUT> mockContext = new MockCoContext<IN1, IN2, OUT>(input1, input2);
-		invokable.setup(mockContext, new ExecutionConfig());
+		invokable.setup(mockContext);
 
 		try {
 			invokable.open(null);
@@ -222,4 +222,9 @@ public class MockCoContext<IN1, IN2, OUT> implements StreamTaskContext<OUT> {
 				"Indexed iterator is currently unsupported for connected streams.");
 	}
 
+	@Override
+	public ExecutionConfig getExecutionConfig() {
+		return new ExecutionConfig();
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
index f3d977f..03038b3 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
@@ -112,7 +112,7 @@ public class MockContext<IN, OUT> implements StreamTaskContext<OUT> {
 	public static <IN, OUT> List<OUT> createAndExecute(StreamInvokable<IN, OUT> invokable,
 			List<IN> inputs) {
 		MockContext<IN, OUT> mockContext = new MockContext<IN, OUT>(inputs);
-		invokable.setup(mockContext, new ExecutionConfig());
+		invokable.setup(mockContext);
 		try {
 			invokable.open(null);
 			invokable.invoke();
@@ -170,4 +170,9 @@ public class MockContext<IN, OUT> implements StreamTaskContext<OUT> {
 		return (IndexedReaderIterator<X>) iterator;
 	}
 
+	@Override
+	public ExecutionConfig getExecutionConfig() {
+		return new ExecutionConfig();
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
index 2fb9345..0b686e5 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j-test.properties
@@ -16,4 +16,12 @@
 # limitations under the License.
 ################################################################################
 
-log4j.rootLogger=OFF
\ No newline at end of file
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=OFF, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/44702075/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j.properties b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j.properties
new file mode 100644
index 0000000..ed2bbcb
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/resources/log4j.properties
@@ -0,0 +1,27 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# This file ensures that tests executed from the IDE show log output
+
+log4j.rootLogger=OFF, console
+
+# Log all infos in the given file
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target = System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
\ No newline at end of file


[12/20] flink git commit: [streaming] Streaming scala api fix for window changes + example cleanup

Posted by mb...@apache.org.
[streaming] Streaming scala api fix for window changes + example cleanup


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/aef52e8e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/aef52e8e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/aef52e8e

Branch: refs/heads/master
Commit: aef52e8eaebc1d609fc650d58d4813c6422dd2e3
Parents: f32990f
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Feb 8 13:46:47 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../api/datastream/DiscretizedStream.java       | 11 ++-
 .../api/datastream/WindowedDataStream.java      | 13 ++--
 .../socket/SocketTextStreamWordCount.java       | 27 +++++---
 .../examples/windowing/TopSpeedWindowing.scala  |  2 +-
 .../flink/streaming/api/scala/DataStream.scala  |  8 +--
 .../api/scala/WindowedDataStream.scala          | 70 ++++++++++++--------
 6 files changed, 77 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/aef52e8e/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
index 37fa1e1..55613b4 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
@@ -67,6 +67,7 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
+	@Override
 	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
 
 		DiscretizedStream<OUT> out = partition(false).transform("Window Reduce", getType(),
@@ -93,12 +94,19 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
+	@Override
 	public <R> DiscretizedStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction) {
 
 		TypeInformation<R> retType = TypeExtractor.getGroupReduceReturnTypes(reduceFunction,
 				getType());
 
-		DiscretizedStream<R> out = partition(true).transform("Window Reduce", retType,
+		return mapWindow(reduceFunction, retType);
+	}
+
+	@Override
+	public <R> DiscretizedStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction,
+			TypeInformation<R> returnType) {
+		DiscretizedStream<R> out = partition(true).transform("Window Reduce", returnType,
 				new WindowMapper<OUT, R>(reduceFunction));
 
 		if (isGrouped()) {
@@ -106,7 +114,6 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		} else {
 			return out;
 		}
-
 	}
 
 	private <R> DiscretizedStream<R> transform(String operatorName, TypeInformation<R> retType,

http://git-wip-us.apache.org/repos/asf/flink/blob/aef52e8e/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index e2cb3f7..7c2dc47 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -240,7 +240,6 @@ public class WindowedDataStream<OUT> {
 	 * @return The transformed DataStream
 	 */
 	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
-
 		return discretize(false).reduceWindow(reduceFunction);
 	}
 
@@ -261,10 +260,6 @@ public class WindowedDataStream<OUT> {
 		return discretize(true).mapWindow(reduceFunction);
 	}
 
-	public DataStream<OUT> flatten() {
-		return dataStream;
-	}
-
 	/**
 	 * Applies a reduceGroup transformation on the windowed data stream by
 	 * reducing the current window at every trigger. In contrast with the
@@ -281,10 +276,14 @@ public class WindowedDataStream<OUT> {
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
-	public <R> SingleOutputStreamOperator<R, ?> mapWindow(
+	public <R> WindowedDataStream<R> mapWindow(
 			GroupReduceFunction<OUT, R> reduceFunction, TypeInformation<R> outType) {
 
-		throw new RuntimeException("Not implemented yet");
+		return discretize(true).mapWindow(reduceFunction, outType);
+	}
+
+	public DataStream<OUT> flatten() {
+		return dataStream;
 	}
 
 	protected Class<?> getClassAtPos(int pos) {

http://git-wip-us.apache.org/repos/asf/flink/blob/aef52e8e/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
index 663541e..e9b60f4 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
@@ -17,13 +17,9 @@
 
 package org.apache.flink.streaming.examples.socket;
 
-import java.util.concurrent.TimeUnit;
-
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.WindowedDataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.windowing.helper.Time;
 import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
 
 /**
@@ -55,19 +51,29 @@ import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
 public class SocketTextStreamWordCount {
 	public static void main(String[] args) throws Exception {
 
+		if (!parseParameters(args)) {
+			return;
+		}
+
 		// set up the execution environment
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment
+				.getExecutionEnvironment();
 
 		// get input data
-		DataStream<String> text = env.socketTextStream("localhost", 9999);
+		DataStream<String> text = env.socketTextStream(hostName, port);
 
-		WindowedDataStream<Tuple2<String, Integer>> counts =
+		DataStream<Tuple2<String, Integer>> counts =
 		// split up the lines in pairs (2-tuples) containing: (word,1)
 		text.flatMap(new Tokenizer())
 		// group by the tuple field "0" and sum up tuple field "1"
-				.window(Time.of(5, TimeUnit.SECONDS)).groupBy(0).sum(1);
+				.groupBy(0)
+				.sum(1);
 
-		counts.print();
+		if (fileOutput) {
+			counts.writeAsText(outputPath, 1);
+		} else {
+			counts.print();
+		}
 
 		// execute program
 		env.execute("WordCount from SocketTextStream Example");
@@ -94,8 +100,7 @@ public class SocketTextStreamWordCount {
 			hostName = args[0];
 			port = Integer.valueOf(args[1]);
 		} else {
-			System.err
-					.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]");
+			System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]");
 			return false;
 		}
 		return true;

http://git-wip-us.apache.org/repos/asf/flink/blob/aef52e8e/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
index c94901d..76f4d2d 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -52,7 +52,7 @@ object TopSpeedWindowing {
       .local    
       .maxBy("speed")
 
-    cars print
+    cars.flatten print
 
     StreamExecutionEnvironment.getExecutionEnvironment.execute("TopSpeedWindowing")
 

http://git-wip-us.apache.org/repos/asf/flink/blob/aef52e8e/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index 23495a5..f6e92aa 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -451,8 +451,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
    * the trigger and eviction policies please use to
    * window(List(triggers), List(evicters))
    */
-  def window(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
-    javaStream.window(windowingHelper: _*)
+  def window(windowingHelper: WindowingHelper[_]): WindowedDataStream[T] =
+    javaStream.window(windowingHelper)
 
   /**
    * Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
@@ -461,8 +461,8 @@ class DataStream[T](javaStream: JavaStream[T]) {
    * use-cases please refer to window(WindowingHelper[_]*)
    *
    */
-  def window(triggers: List[TriggerPolicy[T]], evicters: List[EvictionPolicy[T]]):
-    WindowedDataStream[T] = javaStream.window(triggers, evicters)
+  def window(trigger: TriggerPolicy[T], evicter: EvictionPolicy[T]):
+    WindowedDataStream[T] = javaStream.window(trigger, evicter)
 
   /**
    *

http://git-wip-us.apache.org/repos/asf/flink/blob/aef52e8e/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
index b22e527..5f244d0 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -35,17 +35,18 @@ import org.apache.flink.streaming.api.function.aggregation.SumFunction
 import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.clean
 import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
 import org.apache.flink.streaming.api.windowing.helper._
+import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow
 import org.apache.flink.util.Collector
 
-class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
+class WindowedDataStream[T](javaStream: JavaWStream[T]) {
 
   /**
    * Defines the slide size (trigger frequency) for the windowed data stream.
    * This controls how often the user defined function will be triggered on
    * the window.
    */
-  def every(windowingHelper: WindowingHelper[_]*): WindowedDataStream[T] =
-    javaStream.every(windowingHelper: _*)
+  def every(windowingHelper: WindowingHelper[_]): WindowedDataStream[T] =
+    javaStream.every(windowingHelper)
 
   /**
    * Groups the elements of the WindowedDataStream using the given
@@ -94,20 +95,29 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * operator
    * 
    */
-  def local(): WindowedDataStream[T]= {
-    javaStream.local
-  }
+  def local(): WindowedDataStream[T] = javaStream.local
+ 
+  /**
+   * Flattens the result of a window transformation returning the stream of window
+   * contents elementwise
+   */
+  def flatten(): DataStream[T] = javaStream.flatten()
+  
+  /**
+   * Returns the stream of StreamWindows created by the window tranformation
+   */
+  def getDiscretizedStream(): DataStream[StreamWindow[T]] = javaStream.getDiscretizedStream()
 
   /**
    * Applies a reduce transformation on the windowed data stream by reducing
    * the current window at every trigger.
    *
    */
-  def reduce(reducer: ReduceFunction[T]): DataStream[T] = {
+  def reduceWindow(reducer: ReduceFunction[T]): WindowedDataStream[T] = {
     if (reducer == null) {
       throw new NullPointerException("Reduce function must not be null.")
     }
-    javaStream.reduce(reducer)
+    javaStream.reduceWindow(reducer)
   }
 
   /**
@@ -115,7 +125,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * the current window at every trigger.
    *
    */
-  def reduce(fun: (T, T) => T): DataStream[T] = {
+  def reduceWindow(fun: (T, T) => T): WindowedDataStream[T] = {
     if (fun == null) {
       throw new NullPointerException("Reduce function must not be null.")
     }
@@ -123,7 +133,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
       val cleanFun = clean(fun)
       def reduce(v1: T, v2: T) = { cleanFun(v1, v2) }
     }
-    reduce(reducer)
+    reduceWindow(reducer)
   }
 
   /**
@@ -134,12 +144,12 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * </br>
    * Whenever possible try to use reduce instead of groupReduce for increased efficiency
    */
-  def reduceGroup[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
-  DataStream[R] = {
+  def mapWindow[R: ClassTag: TypeInformation](reducer: GroupReduceFunction[T, R]):
+  WindowedDataStream[R] = {
     if (reducer == null) {
       throw new NullPointerException("GroupReduce function must not be null.")
     }
-    javaStream.reduceGroup(reducer, implicitly[TypeInformation[R]])
+    javaStream.mapWindow(reducer, implicitly[TypeInformation[R]])
   }
 
   /**
@@ -150,8 +160,8 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * </br>
    * Whenever possible try to use reduce instead of groupReduce for increased efficiency
    */
-  def reduceGroup[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
-  DataStream[R] = {
+  def mapWindow[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
+  WindowedDataStream[R] = {
     if (fun == null) {
       throw new NullPointerException("GroupReduce function must not be null.")
     }
@@ -159,7 +169,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
       val cleanFun = clean(fun)
       def reduce(in: java.lang.Iterable[T], out: Collector[R]) = { cleanFun(in, out) }
     }
-    reduceGroup(reducer)
+    mapWindow(reducer)
   }
 
   /**
@@ -167,47 +177,47 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * the given position.
    *
    */
-  def max(position: Int): DataStream[T] = aggregate(AggregationType.MAX, position)
+  def max(position: Int): WindowedDataStream[T] = aggregate(AggregationType.MAX, position)
   
   /**
    * Applies an aggregation that that gives the maximum of the elements in the window at
    * the given field.
    *
    */
-  def max(field: String): DataStream[T] = aggregate(AggregationType.MAX, field)
+  def max(field: String): WindowedDataStream[T] = aggregate(AggregationType.MAX, field)
 
   /**
    * Applies an aggregation that that gives the minimum of the elements in the window at
    * the given position.
    *
    */
-  def min(position: Int): DataStream[T] = aggregate(AggregationType.MIN, position)
+  def min(position: Int): WindowedDataStream[T] = aggregate(AggregationType.MIN, position)
   
   /**
    * Applies an aggregation that that gives the minimum of the elements in the window at
    * the given field.
    *
    */
-  def min(field: String): DataStream[T] = aggregate(AggregationType.MIN, field)
+  def min(field: String): WindowedDataStream[T] = aggregate(AggregationType.MIN, field)
 
   /**
    * Applies an aggregation that sums the elements in the window at the given position.
    *
    */
-  def sum(position: Int): DataStream[T] = aggregate(AggregationType.SUM, position)
+  def sum(position: Int): WindowedDataStream[T] = aggregate(AggregationType.SUM, position)
   
   /**
    * Applies an aggregation that sums the elements in the window at the given field.
    *
    */
-  def sum(field: String): DataStream[T] = aggregate(AggregationType.SUM, field)
+  def sum(field: String): WindowedDataStream[T] = aggregate(AggregationType.SUM, field)
 
   /**
    * Applies an aggregation that that gives the maximum element of the window by
    * the given position. When equality, returns the first.
    *
    */
-  def maxBy(position: Int): DataStream[T] = aggregate(AggregationType.MAXBY,
+  def maxBy(position: Int): WindowedDataStream[T] = aggregate(AggregationType.MAXBY,
     position)
     
   /**
@@ -215,7 +225,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * the given field. When equality, returns the first.
    *
    */
-  def maxBy(field: String): DataStream[T] = aggregate(AggregationType.MAXBY,
+  def maxBy(field: String): WindowedDataStream[T] = aggregate(AggregationType.MAXBY,
     field)
 
   /**
@@ -223,7 +233,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * the given position. When equality, returns the first.
    *
    */
-  def minBy(position: Int): DataStream[T] = aggregate(AggregationType.MINBY,
+  def minBy(position: Int): WindowedDataStream[T] = aggregate(AggregationType.MINBY,
     position)
     
    /**
@@ -231,16 +241,17 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
    * the given field. When equality, returns the first.
    *
    */
-  def minBy(field: String): DataStream[T] = aggregate(AggregationType.MINBY,
+  def minBy(field: String): WindowedDataStream[T] = aggregate(AggregationType.MINBY,
     field)
     
-  private def aggregate(aggregationType: AggregationType, field: String): DataStream[T] = {
+  private def aggregate(aggregationType: AggregationType, field: String): 
+  WindowedDataStream[T] = {
     val position = fieldNames2Indices(javaStream.getType(), Array(field))(0)
     aggregate(aggregationType, position)
   }  
 
   def aggregate(aggregationType: AggregationType, position: Int):
-  DataStream[T] = {
+  WindowedDataStream[T] = {
 
     val jStream = javaStream.asInstanceOf[JavaWStream[Product]]
     val outType = jStream.getType().asInstanceOf[TupleTypeInfoBase[_]]
@@ -256,7 +267,8 @@ class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
       case _ => new agg.ProductComparableAggregator(aggregationType, true)
     }
 
-    new DataStream[Product](jStream.reduce(reducer)).asInstanceOf[DataStream[T]]
+    new WindowedDataStream[Product](
+            jStream.reduceWindow(reducer)).asInstanceOf[WindowedDataStream[T]]
   }
 
 }


[09/20] flink git commit: [streaming] Test cases added for all new windowing invokables

Posted by mb...@apache.org.
[streaming] Test cases added for all new windowing invokables


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3d10a235
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3d10a235
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3d10a235

Branch: refs/heads/master
Commit: 3d10a2352e41deb46b72fab5a07921a49464cf29
Parents: f571ece
Author: Gyula Fora <gy...@apache.org>
Authored: Sat Feb 14 20:36:45 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../streaming/api/windowing/StreamWindow.java   |  3 +
 .../operator/windowing/WindowFlattenerTest.java | 53 ++++++++++++++
 .../windowing/WindowIntegrationTest.java        |  2 +-
 .../operator/windowing/WindowMapperTest.java    | 60 ++++++++++++++++
 .../operator/windowing/WindowMergerTest.java    | 75 ++++++++++++++++++++
 .../windowing/WindowPartitionerTest.java        | 75 ++++++++++++++++++++
 .../operator/windowing/WindowReducerTest.java   | 61 ++++++++++++++++
 7 files changed, 328 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/3d10a235/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
index 74ed62e..49b40f5 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
@@ -140,6 +140,9 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 	 */
 	public List<StreamWindow<T>> split(int n) {
 		int numElements = size();
+		if (n == 0) {
+			return new ArrayList<StreamWindow<T>>();
+		}
 		if (n > numElements) {
 			return split(numElements);
 		} else {

http://git-wip-us.apache.org/repos/asf/flink/blob/3d10a235/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattenerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattenerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattenerTest.java
new file mode 100644
index 0000000..bee6733
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattenerTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class WindowFlattenerTest {
+
+	@Test
+	public void test() {
+		StreamInvokable<StreamWindow<Integer>, Integer> flattener = new WindowFlattener<Integer>();
+
+		StreamWindow<Integer> w1 = StreamWindow.fromElements(1, 2, 3);
+		StreamWindow<Integer> w2 = new StreamWindow<Integer>();
+
+		List<StreamWindow<Integer>> input = new ArrayList<StreamWindow<Integer>>();
+		input.add(w1);
+		input.add(w2);
+
+		List<Integer> expected = new ArrayList<Integer>();
+		expected.addAll(w1);
+		expected.addAll(w2);
+
+		List<Integer> output = MockContext.createAndExecute(flattener, input);
+
+		assertEquals(expected, output);
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3d10a235/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
index a430700..05ac5fd 100755
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
@@ -147,7 +147,7 @@ public class WindowIntegrationTest implements Serializable {
 
 	}
 
-	private <R> void validateOutput(List<R> expected, List<R> actual) {
+	public static <R> void validateOutput(List<R> expected, List<R> actual) {
 		assertEquals(new HashSet<R>(expected), new HashSet<R>(actual));
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/3d10a235/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapperTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapperTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapperTest.java
new file mode 100644
index 0000000..c4022ea
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapperTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.WindowMapFunction;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.util.MockContext;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class WindowMapperTest {
+
+	@Test
+	public void test() {
+		StreamInvokable<StreamWindow<Integer>, StreamWindow<Integer>> windowMapper = new WindowMapper<Integer, Integer>(
+				new WindowMapFunction<Integer, Integer>() {
+
+					private static final long serialVersionUID = 1L;
+
+					@Override
+					public void mapWindow(Iterable<Integer> values, Collector<Integer> out)
+							throws Exception {
+						for (Integer v : values) {
+							out.collect(v);
+						}
+					}
+				});
+
+		List<StreamWindow<Integer>> input = new ArrayList<StreamWindow<Integer>>();
+		input.add(StreamWindow.fromElements(1, 2, 3));
+		input.add(new StreamWindow<Integer>());
+
+		List<StreamWindow<Integer>> output = MockContext.createAndExecute(windowMapper, input);
+
+		assertEquals(input, output);
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3d10a235/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMergerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMergerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMergerTest.java
new file mode 100644
index 0000000..ac3f583
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMergerTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class WindowMergerTest {
+
+	@Test
+	public void test() throws Exception {
+		StreamInvokable<StreamWindow<Integer>, StreamWindow<Integer>> windowMerger = new WindowMerger<Integer>();
+
+		StreamWindow<Integer> w1 = new StreamWindow<Integer>();
+		StreamWindow<Integer> w2 = StreamWindow.fromElements(1, 2, 3, 4);
+		StreamWindow<Integer> w3 = StreamWindow.fromElements(-1, 2, 3, 4);
+		StreamWindow<Integer> w4_1 = new StreamWindow<Integer>(1, 2);
+		StreamWindow<Integer> w4_2 = new StreamWindow<Integer>(1, 2);
+		w4_1.add(1);
+		w4_2.add(2);
+
+		List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
+		expected.add(w1);
+		expected.add(w2);
+		expected.add(w3);
+		expected.add(StreamWindow.fromElements(1, 2));
+
+		List<StreamWindow<Integer>> input = new ArrayList<StreamWindow<Integer>>();
+		input.add(w1);
+		input.add(w4_1);
+		input.addAll(w2.split(2));
+		input.addAll(w3.partitionBy(new KeySelector<Integer, Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer getKey(Integer value) throws Exception {
+				return value % 2;
+			}
+		}));
+		input.add(w4_2);
+
+		List<StreamWindow<Integer>> output = MockContext.createAndExecute(windowMerger, input);
+
+		assertEquals(expected.size(), expected.size());
+		for (int i = 0; i < output.size(); i++) {
+			assertEquals(new HashSet<Integer>(expected.get(i)), new HashSet<Integer>(output.get(i)));
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3d10a235/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitionerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitionerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitionerTest.java
new file mode 100644
index 0000000..da68211
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitionerTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class WindowPartitionerTest {
+
+	@Test
+	public void test() throws Exception {
+		StreamInvokable<StreamWindow<Integer>, StreamWindow<Integer>> splitPartitioner = new WindowPartitioner<Integer>(
+				2);
+
+		StreamInvokable<StreamWindow<Integer>, StreamWindow<Integer>> gbPartitioner = new WindowPartitioner<Integer>(
+				new MyKey());
+
+		StreamWindow<Integer> w1 = new StreamWindow<Integer>();
+		StreamWindow<Integer> w2 = StreamWindow.fromElements(1, 2, 3, 4);
+
+		List<StreamWindow<Integer>> expected1 = new ArrayList<StreamWindow<Integer>>();
+		expected1.addAll(w1.split(2));
+		expected1.addAll(w2.split(2));
+
+		List<StreamWindow<Integer>> expected2 = new ArrayList<StreamWindow<Integer>>();
+		expected2.addAll(w1.partitionBy(new MyKey()));
+		expected2.addAll(w2.partitionBy(new MyKey()));
+
+		List<StreamWindow<Integer>> input = new ArrayList<StreamWindow<Integer>>();
+		input.add(w1);
+		input.add(w2);
+
+		List<StreamWindow<Integer>> output1 = MockContext.createAndExecute(splitPartitioner, input);
+		List<StreamWindow<Integer>> output2 = MockContext.createAndExecute(gbPartitioner, input);
+
+		assertEquals(expected1, output1);
+		assertEquals(expected2, output2);
+
+	}
+
+	private static class MyKey implements KeySelector<Integer, Object> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public Object getKey(Integer value) throws Exception {
+			return value / 2;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/3d10a235/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducerTest.java
new file mode 100644
index 0000000..e3d742c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducerTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class WindowReducerTest {
+
+	@Test
+	public void test() {
+		StreamInvokable<StreamWindow<Integer>, StreamWindow<Integer>> windowReducer = new WindowReducer<Integer>(
+				new ReduceFunction<Integer>() {
+
+					private static final long serialVersionUID = 1L;
+
+					@Override
+					public Integer reduce(Integer value1, Integer value2) throws Exception {
+						return value1 + value2;
+					}
+				});
+
+		List<StreamWindow<Integer>> input = new ArrayList<StreamWindow<Integer>>();
+		input.add(StreamWindow.fromElements(1, 2, 3));
+		input.add(new StreamWindow<Integer>());
+		input.add(StreamWindow.fromElements(-1));
+
+		List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
+		expected.add(StreamWindow.fromElements(6));
+		expected.add(new StreamWindow<Integer>());
+		expected.add(StreamWindow.fromElements(-1));
+
+		List<StreamWindow<Integer>> output = MockContext.createAndExecute(windowReducer, input);
+
+		assertEquals(expected, output);
+
+	}
+}


[06/20] flink git commit: [streaming] StreamWindow abstraction added with typeinfo and tests

Posted by mb...@apache.org.
[streaming] StreamWindow abstraction added with typeinfo and tests


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/667f8199
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/667f8199
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/667f8199

Branch: refs/heads/master
Commit: 667f819913ec59a433c51907132f5eca4000e002
Parents: 26ae979
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Feb 4 17:14:51 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:07 2015 +0100

----------------------------------------------------------------------
 .../operator/windowing/StreamWindow.java        | 158 ++++++++++++++++++
 .../windowing/StreamWindowSerializer.java       | 136 ++++++++++++++++
 .../windowing/StreamWindowTypeInfo.java         |  72 +++++++++
 .../operator/windowing/StreamWindowTest.java    | 161 +++++++++++++++++++
 4 files changed, 527 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/667f8199/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
new file mode 100644
index 0000000..b37bf47
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.util.Collector;
+
+import com.google.common.collect.Lists;
+
+public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
+
+	private static final long serialVersionUID = -5150196421193988403L;
+	private static Random rnd = new Random();
+
+	public int windowID;
+	public int transformationID;
+
+	public int numberOfParts;
+
+	public StreamWindow() {
+		this(rnd.nextInt(), rnd.nextInt(), 1);
+	}
+
+	public StreamWindow(int windowID) {
+		this(windowID, rnd.nextInt(), 1);
+	}
+
+	public StreamWindow(int windowID, int transformationID, int numberOfParts) {
+		super();
+		this.windowID = windowID;
+		this.transformationID = transformationID;
+		this.numberOfParts = numberOfParts;
+	}
+
+	public StreamWindow(StreamWindow<T> window) {
+		this(window.windowID, window.transformationID, window.numberOfParts);
+		addAll(window);
+	}
+
+	public StreamWindow(StreamWindow<T> window, TypeSerializer<T> serializer) {
+		this(window.windowID, window.transformationID, window.numberOfParts);
+		for (T element : window) {
+			add(serializer.copy(element));
+		}
+	}
+
+	public List<StreamWindow<T>> partitionBy(KeySelector<T, ?> keySelector) throws Exception {
+		Map<Object, StreamWindow<T>> partitions = new HashMap<Object, StreamWindow<T>>();
+
+		for (T value : this) {
+			Object key = keySelector.getKey(value);
+			StreamWindow<T> window = partitions.get(key);
+			if (window == null) {
+				window = new StreamWindow<T>(this.windowID, this.transformationID, 0);
+				partitions.put(key, window);
+			}
+			window.add(value);
+		}
+
+		List<StreamWindow<T>> output = new ArrayList<StreamWindow<T>>();
+		int numkeys = partitions.size();
+
+		for (StreamWindow<T> window : partitions.values()) {
+			output.add(window.setNumberOfParts(numkeys));
+		}
+
+		return output;
+	}
+
+	public List<StreamWindow<T>> split(int n) {
+		List<List<T>> subLists = Lists.partition(this, (int) Math.ceil((double) size() / n));
+		List<StreamWindow<T>> split = new ArrayList<StreamWindow<T>>(n);
+		for (List<T> partition : subLists) {
+			StreamWindow<T> subWindow = new StreamWindow<T>(windowID, transformationID,
+					subLists.size());
+			subWindow.addAll(partition);
+			split.add(subWindow);
+		}
+		return split;
+	}
+
+	public StreamWindow<T> setNumberOfParts(int n) {
+		this.numberOfParts = n;
+		return this;
+	}
+
+	public boolean compatibleWith(StreamWindow<T> otherWindow) {
+		return this.windowID == otherWindow.windowID && this.numberOfParts > 1;
+	}
+
+	public static <R> StreamWindow<R> merge(StreamWindow<R>... windows) {
+		StreamWindow<R> window = new StreamWindow<R>(windows[0]);
+		for (int i = 1; i < windows.length; i++) {
+			StreamWindow<R> next = windows[i];
+			if (window.compatibleWith(next)) {
+				window.addAll(next);
+				window.numberOfParts--;
+			} else {
+				throw new RuntimeException("Can only merge compatible windows");
+			}
+		}
+		return window;
+	}
+
+	public static <R> StreamWindow<R> merge(List<StreamWindow<R>> windows) {
+		if (windows.isEmpty()) {
+			throw new RuntimeException("Need at least one window to merge");
+		} else {
+			StreamWindow<R> window = new StreamWindow<R>(windows.get(0));
+			for (int i = 1; i < windows.size(); i++) {
+				StreamWindow<R> next = windows.get(i);
+				if (window.compatibleWith(next)) {
+					window.addAll(next);
+					window.numberOfParts--;
+				} else {
+					throw new RuntimeException("Can only merge compatible windows");
+				}
+			}
+			return window;
+		}
+	}
+
+	@Override
+	public void collect(T record) {
+		add(record);
+	}
+
+	@Override
+	public void close() {
+	}
+
+	@Override
+	public String toString() {
+		return super.toString() + " " + windowID + " (" + numberOfParts + ")";
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/667f8199/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
new file mode 100755
index 0000000..70c608c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.io.IOException;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+public final class StreamWindowSerializer<T> extends TypeSerializer<StreamWindow<T>> {
+
+	private static final long serialVersionUID = 1L;
+
+	private final TypeSerializer<T> typeSerializer;
+	TypeSerializer<Integer> intSerializer = BasicTypeInfo.INT_TYPE_INFO.createSerializer();
+
+	public StreamWindowSerializer(TypeInformation<T> typeInfo) {
+		this.typeSerializer = typeInfo.createSerializer();
+	}
+
+	public TypeSerializer<T> getObjectSerializer() {
+		return typeSerializer;
+	}
+
+	@Override
+	public boolean isImmutableType() {
+		return false;
+	}
+
+	@Override
+	public boolean isStateful() {
+		return false;
+	}
+
+	@Override
+	public StreamWindow<T> createInstance() {
+		return new StreamWindow<T>(0, 0, 0);
+	}
+
+	@Override
+	public StreamWindow<T> copy(StreamWindow<T> from) {
+		return new StreamWindow<T>(from, typeSerializer);
+	}
+
+	@Override
+	public StreamWindow<T> copy(StreamWindow<T> from, StreamWindow<T> reuse) {
+		reuse.clear();
+		reuse.windowID = from.windowID;
+		reuse.transformationID = from.transformationID;
+		reuse.numberOfParts = from.numberOfParts;
+
+		for (T element : from) {
+			reuse.add(typeSerializer.copy(element));
+		}
+		return reuse;
+	}
+
+	@Override
+	public int getLength() {
+		return -1;
+	}
+
+	@Override
+	public void serialize(StreamWindow<T> value, DataOutputView target) throws IOException {
+
+		intSerializer.serialize(value.windowID, target);
+		intSerializer.serialize(value.transformationID, target);
+		intSerializer.serialize(value.numberOfParts, target);
+		intSerializer.serialize(value.size(), target);
+
+		for (T element : value) {
+			typeSerializer.serialize(element, target);
+		}
+	}
+
+	@Override
+	public StreamWindow<T> deserialize(DataInputView source) throws IOException {
+		StreamWindow<T> window = createInstance();
+
+		window.windowID = intSerializer.deserialize(source);
+		window.transformationID = intSerializer.deserialize(source);
+		window.numberOfParts = intSerializer.deserialize(source);
+
+		int size = intSerializer.deserialize(source);
+
+		for (int i = 0; i < size; i++) {
+			window.add(typeSerializer.deserialize(source));
+		}
+
+		return window;
+	}
+
+	@Override
+	public StreamWindow<T> deserialize(StreamWindow<T> reuse, DataInputView source)
+			throws IOException {
+
+		StreamWindow<T> window = reuse;
+		window.clear();
+
+		window.windowID = source.readInt();
+		window.transformationID = source.readInt();
+		window.numberOfParts = source.readInt();
+
+		int size = source.readInt();
+
+		for (int i = 0; i < size; i++) {
+			window.add(typeSerializer.deserialize(source));
+		}
+
+		return window;
+	}
+
+	@Override
+	public void copy(DataInputView source, DataOutputView target) throws IOException {
+		// Needs to be implemented
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/667f8199/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
new file mode 100644
index 0000000..a9a4bba
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTypeInfo.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+
+public class StreamWindowTypeInfo<T> extends TypeInformation<StreamWindow<T>> {
+
+	TypeInformation<T> innerType;
+
+	public StreamWindowTypeInfo(TypeInformation<T> innerType) {
+		this.innerType = innerType;
+	}
+
+	public TypeInformation<T> getInnerType() {
+		return innerType;
+	}
+
+	@Override
+	public boolean isBasicType() {
+		return innerType.isBasicType();
+	}
+
+	@Override
+	public boolean isTupleType() {
+		return innerType.isTupleType();
+	}
+
+	@Override
+	public int getArity() {
+		return innerType.getArity();
+	}
+
+	@Override
+	public Class<StreamWindow<T>> getTypeClass() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public boolean isKeyType() {
+		return innerType.isKeyType();
+	}
+
+	@Override
+	public TypeSerializer<StreamWindow<T>> createSerializer() {
+		return new StreamWindowSerializer<T>(innerType);
+	}
+
+	@Override
+	public int getTotalFields() {
+		return innerType.getTotalFields();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/667f8199/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
new file mode 100644
index 0000000..66a11f5
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.junit.Test;
+
+public class StreamWindowTest {
+
+	@Test
+	public void creationTest() {
+
+		StreamWindow<Integer> window1 = new StreamWindow<Integer>();
+		assertTrue(window1.isEmpty());
+		assertTrue(window1.windowID != 0);
+
+		window1.add(10);
+		assertEquals(1, window1.size());
+
+		StreamWindow<Integer> window2 = new StreamWindow<Integer>(window1);
+
+		assertTrue(window1.windowID == window2.windowID);
+		assertEquals(1, window2.size());
+
+		StreamWindow<Integer> window3 = new StreamWindow<Integer>(100);
+		assertEquals(100, window3.windowID);
+
+		StreamWindow<Integer> window4 = new StreamWindow<Integer>();
+		assertFalse(window4.windowID == window1.windowID);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void mergeTest() throws IOException {
+		StreamWindow<Integer> window1 = new StreamWindow<Integer>().setNumberOfParts(3);
+		StreamWindow<Integer> window2 = new StreamWindow<Integer>(window1.windowID,
+				window1.transformationID, 3);
+		StreamWindow<Integer> window3 = new StreamWindow<Integer>(window1.windowID,
+				window1.transformationID, 3);
+
+		window1.add(1);
+		window2.add(2);
+		window3.add(3);
+
+		Set<Integer> values = new HashSet<Integer>();
+		values.add(1);
+		values.add(2);
+		values.add(3);
+
+		StreamWindow<Integer> merged = StreamWindow.merge(window1, window2, window3);
+
+		assertEquals(3, merged.size());
+		assertEquals(window1.windowID, merged.windowID);
+		assertEquals(values, new HashSet<Integer>(merged));
+
+		try {
+			StreamWindow.merge(window1, new StreamWindow<Integer>());
+			fail();
+		} catch (RuntimeException e) {
+			// good
+		}
+
+		List<StreamWindow<Integer>> wList = merged.split(3);
+
+		StreamWindow<Integer> merged2 = StreamWindow.merge(wList);
+
+		assertEquals(3, merged2.size());
+		assertEquals(window1.windowID, merged2.windowID);
+		assertEquals(values, new HashSet<Integer>(merged2));
+
+		TypeSerializer<StreamWindow<Integer>> ts = new StreamWindowSerializer<Integer>(
+				BasicTypeInfo.INT_TYPE_INFO);
+
+		TestOutputView ow = new TestOutputView();
+
+		ts.serialize(merged2, ow);
+
+		TestInputView iw = ow.getInputView();
+
+		System.out.println(ts.deserialize(iw));
+
+	}
+
+	@Test
+	public void partitionTest() {
+
+	}
+
+	private class TestOutputView extends DataOutputStream implements DataOutputView {
+
+		public TestOutputView() {
+			super(new ByteArrayOutputStream(4096));
+		}
+
+		public TestInputView getInputView() {
+			ByteArrayOutputStream baos = (ByteArrayOutputStream) out;
+			return new TestInputView(baos.toByteArray());
+		}
+
+		@Override
+		public void skipBytesToWrite(int numBytes) throws IOException {
+			for (int i = 0; i < numBytes; i++) {
+				write(0);
+			}
+		}
+
+		@Override
+		public void write(DataInputView source, int numBytes) throws IOException {
+			byte[] buffer = new byte[numBytes];
+			source.readFully(buffer);
+			write(buffer);
+		}
+	}
+
+	private class TestInputView extends DataInputStream implements DataInputView {
+
+		public TestInputView(byte[] data) {
+			super(new ByteArrayInputStream(data));
+		}
+
+		@Override
+		public void skipBytesToRead(int numBytes) throws IOException {
+			while (numBytes > 0) {
+				int skipped = skipBytes(numBytes);
+				numBytes -= skipped;
+			}
+		}
+	}
+}


[14/20] flink git commit: [streaming] WindowBuffer interface added for preaggregator logic + simple tumbling prereducer

Posted by mb...@apache.org.
[streaming] WindowBuffer interface added for preaggregator logic + simple tumbling prereducer


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ef7b7cd9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ef7b7cd9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ef7b7cd9

Branch: refs/heads/master
Commit: ef7b7cd968c13ce9df15e30e33300e0f26def303
Parents: 8708688
Author: Gyula Fora <gy...@apache.org>
Authored: Tue Feb 10 12:21:09 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../api/datastream/DiscretizedStream.java       | 47 ++++++-----
 .../api/datastream/WindowedDataStream.java      | 83 +++++++++++++++-----
 .../operator/StreamReduceInvokable.java         |  2 +-
 .../operator/windowing/BasicWindowBuffer.java   | 74 +++++++++++++++++
 .../windowing/GroupedStreamDiscretizer.java     | 10 ++-
 .../windowing/GroupedTimeDiscretizer.java       | 27 +------
 .../operator/windowing/StreamDiscretizer.java   | 54 ++++---------
 .../windowing/StreamWindowSerializer.java       | 10 +--
 .../operator/windowing/TumblingPreReducer.java  | 78 ++++++++++++++++++
 .../operator/windowing/WindowBuffer.java        | 36 +++++++++
 .../windowing/GroupedStreamDiscretizerTest.java |  2 +-
 .../windowing/StreamDiscretizerTest.java        |  6 +-
 .../windowing/WindowIntegrationTest.java        |  2 +
 .../examples/windowing/StockPrices.java         | 27 +++++--
 .../scala/examples/windowing/StockPrices.scala  | 29 ++++---
 .../flink/streaming/api/scala/package.scala     |  6 +-
 16 files changed, 360 insertions(+), 133 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
index 55613b4..5541ba3 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
@@ -28,11 +28,11 @@ import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.typeutils.TupleTypeInfo;
 import org.apache.flink.api.java.typeutils.TypeExtractor;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.windowing.WindowFlattener;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
-import org.apache.flink.streaming.api.invokable.operator.windowing.WindowMerger;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowFlattener;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowMapper;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowMerger;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowPartitioner;
 import org.apache.flink.streaming.api.invokable.operator.windowing.WindowReducer;
 
@@ -48,13 +48,15 @@ import org.apache.flink.streaming.api.invokable.operator.windowing.WindowReducer
  */
 public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 
-	protected SingleOutputStreamOperator<StreamWindow<OUT>, ?> discretizedStream;
+	private SingleOutputStreamOperator<StreamWindow<OUT>, ?> discretizedStream;
+	private WindowTransformation transformation;
 
 	protected DiscretizedStream(SingleOutputStreamOperator<StreamWindow<OUT>, ?> discretizedStream,
-			KeySelector<OUT, ?> groupByKey) {
+			KeySelector<OUT, ?> groupByKey, WindowTransformation tranformation) {
 		super();
 		this.groupByKey = groupByKey;
 		this.discretizedStream = discretizedStream;
+		this.transformation = tranformation;
 	}
 
 	/**
@@ -70,12 +72,13 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 	@Override
 	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
 
-		DiscretizedStream<OUT> out = partition(false).transform("Window Reduce", getType(),
+		DiscretizedStream<OUT> out = partition(transformation).transform(
+				WindowTransformation.REDUCEWINDOW, "Window Reduce", getType(),
 				new WindowReducer<OUT>(reduceFunction)).merge();
 
 		if (!isGrouped()) {
-			return out.transform("Window Reduce", out.getType(), new WindowReducer<OUT>(
-					reduceFunction));
+			return out.transform(WindowTransformation.REDUCEWINDOW, "Window Reduce", out.getType(),
+					new WindowReducer<OUT>(reduceFunction));
 		} else {
 			return out;
 		}
@@ -106,7 +109,8 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 	@Override
 	public <R> DiscretizedStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction,
 			TypeInformation<R> returnType) {
-		DiscretizedStream<R> out = partition(true).transform("Window Reduce", returnType,
+		DiscretizedStream<R> out = partition(transformation).transform(
+				WindowTransformation.REDUCEWINDOW, "Window Reduce", returnType,
 				new WindowMapper<OUT, R>(reduceFunction));
 
 		if (isGrouped()) {
@@ -116,30 +120,31 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		}
 	}
 
-	private <R> DiscretizedStream<R> transform(String operatorName, TypeInformation<R> retType,
+	private <R> DiscretizedStream<R> transform(WindowTransformation transformation,
+			String operatorName, TypeInformation<R> retType,
 			StreamInvokable<StreamWindow<OUT>, StreamWindow<R>> invokable) {
 
 		return wrap(discretizedStream.transform(operatorName, new StreamWindowTypeInfo<R>(retType),
-				invokable));
+				invokable), transformation);
 	}
 
-	private DiscretizedStream<OUT> partition(boolean isMap) {
+	private DiscretizedStream<OUT> partition(WindowTransformation transformation) {
 
 		int parallelism = discretizedStream.getParallelism();
 
 		if (isGrouped()) {
-			DiscretizedStream<OUT> out = transform("Window partitioner", getType(),
+			DiscretizedStream<OUT> out = transform(transformation, "Window partitioner", getType(),
 					new WindowPartitioner<OUT>(groupByKey)).setParallelism(parallelism);
 
 			out.groupByKey = null;
 
 			return out;
-		} else if (!isMap) {
+		} else if (transformation == WindowTransformation.MAPWINDOW) {
 			return transform(
+					transformation,
 					"Window partitioner",
 					getType(),
-					new WindowPartitioner<OUT>(discretizedStream.environment
-							.getDegreeOfParallelism())).setParallelism(parallelism);
+					new WindowPartitioner<OUT>(parallelism)).setParallelism(parallelism);
 		} else {
 			return this;
 		}
@@ -160,9 +165,15 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		return discretizedStream.transform("Window Flatten", getType(), new WindowFlattener<OUT>());
 	}
 
-	@SuppressWarnings({ "unchecked", "rawtypes" })
+	@SuppressWarnings("rawtypes")
 	private <R> DiscretizedStream<R> wrap(SingleOutputStreamOperator stream) {
-		return new DiscretizedStream<R>(stream, (KeySelector<R, ?>) this.groupByKey);
+		return wrap(stream, transformation);
+	}
+
+	@SuppressWarnings({ "unchecked", "rawtypes" })
+	private <R> DiscretizedStream<R> wrap(SingleOutputStreamOperator stream,
+			WindowTransformation transformation) {
+		return new DiscretizedStream<R>(stream, (KeySelector<R, ?>) this.groupByKey, transformation);
 	}
 
 	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
@@ -218,7 +229,7 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 	}
 
 	protected DiscretizedStream<OUT> copy() {
-		return new DiscretizedStream<OUT>(discretizedStream.copy(), groupByKey);
+		return new DiscretizedStream<OUT>(discretizedStream.copy(), groupByKey, transformation);
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 7a214fe..8fff958 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -17,6 +17,7 @@
 
 package org.apache.flink.streaming.api.datastream;
 
+import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
@@ -31,11 +32,14 @@ import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.A
 import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
 import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.windowing.BasicWindowBuffer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedStreamDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedTimeDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
+import org.apache.flink.streaming.api.invokable.operator.windowing.TumblingPreReducer;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowBuffer;
 import org.apache.flink.streaming.api.windowing.helper.Time;
 import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
@@ -210,35 +214,75 @@ public class WindowedDataStream<OUT> {
 		return out;
 	}
 
-	private DiscretizedStream<OUT> discretize(boolean isMap) {
+	private DiscretizedStream<OUT> discretize(WindowTransformation transformation) {
 
-		StreamInvokable<OUT, StreamWindow<OUT>> discretizer = getDiscretizer();
+		StreamInvokable<OUT, StreamWindow<OUT>> discretizer = getDiscretizer(transformation,
+				getTrigger(), getEviction(), discretizerKey);
 
-		int parallelism = isLocal || (discretizerKey != null) ? dataStream.environment
-				.getDegreeOfParallelism() : 1;
+		int parallelism = getDiscretizerParallelism();
 
 		return new DiscretizedStream<OUT>(dataStream.transform("Stream Discretizer",
 				new StreamWindowTypeInfo<OUT>(getType()), discretizer).setParallelism(parallelism),
-				groupByKey);
+				groupByKey, transformation);
+
+	}
+
+	protected enum WindowTransformation {
+
+		REDUCEWINDOW, MAPWINDOW, NONE;
+
+		private Function UDF;
+
+		public WindowTransformation with(Function UDF) {
+			this.UDF = UDF;
+			return this;
+		}
+	}
 
+	private int getDiscretizerParallelism() {
+		return isLocal || (discretizerKey != null) ? dataStream.environment
+				.getDegreeOfParallelism() : 1;
 	}
 
-	private StreamInvokable<OUT, StreamWindow<OUT>> getDiscretizer() {
+	private StreamInvokable<OUT, StreamWindow<OUT>> getDiscretizer(
+			WindowTransformation transformation, TriggerPolicy<OUT> trigger,
+			EvictionPolicy<OUT> eviction, KeySelector<OUT, ?> discretizerKey) {
+
+		WindowBuffer<OUT> windowBuffer = getWindowBuffer(transformation, trigger, eviction,
+				discretizerKey);
+
 		if (discretizerKey == null) {
-			return new StreamDiscretizer<OUT>(getTrigger(), getEvicter());
-		} else if (getTrigger() instanceof TimeTriggerPolicy
-				&& ((TimeTriggerPolicy<OUT>) getTrigger()).timestampWrapper.isDefaultTimestamp()) {
+			return new StreamDiscretizer<OUT>(trigger, eviction, windowBuffer);
+		} else if (trigger instanceof TimeTriggerPolicy
+				&& ((TimeTriggerPolicy<OUT>) trigger).timestampWrapper.isDefaultTimestamp()) {
 			return new GroupedTimeDiscretizer<OUT>(discretizerKey,
-					(TimeTriggerPolicy<OUT>) getTrigger(),
-					(CloneableEvictionPolicy<OUT>) getEvicter());
+					(TimeTriggerPolicy<OUT>) trigger, (CloneableEvictionPolicy<OUT>) eviction,
+					windowBuffer);
 		} else {
 			return new GroupedStreamDiscretizer<OUT>(discretizerKey,
-					(CloneableTriggerPolicy<OUT>) getTrigger(),
-					(CloneableEvictionPolicy<OUT>) getEvicter());
+					(CloneableTriggerPolicy<OUT>) trigger, (CloneableEvictionPolicy<OUT>) eviction,
+					windowBuffer);
 		}
 
 	}
 
+	@SuppressWarnings("unchecked")
+	private WindowBuffer<OUT> getWindowBuffer(WindowTransformation transformation,
+			TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> eviction,
+			KeySelector<OUT, ?> discretizerKey) {
+
+		if (transformation == WindowTransformation.REDUCEWINDOW
+				&& eviction instanceof TumblingEvictionPolicy) {
+			if (discretizerKey == null) {
+				return new TumblingPreReducer<OUT>((ReduceFunction<OUT>) transformation.UDF,
+						getType().createSerializer());
+			} else {
+				return new BasicWindowBuffer<OUT>();
+			}
+		}
+		return new BasicWindowBuffer<OUT>();
+	}
+
 	/**
 	 * Applies a reduce transformation on the windowed data stream by reducing
 	 * the current window at every trigger.The user can also extend the
@@ -250,7 +294,8 @@ public class WindowedDataStream<OUT> {
 	 * @return The transformed DataStream
 	 */
 	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
-		return discretize(false).reduceWindow(reduceFunction);
+		return discretize(WindowTransformation.REDUCEWINDOW.with(reduceFunction)).reduceWindow(
+				reduceFunction);
 	}
 
 	/**
@@ -267,7 +312,8 @@ public class WindowedDataStream<OUT> {
 	 * @return The transformed DataStream
 	 */
 	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction) {
-		return discretize(true).mapWindow(reduceFunction);
+		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction)).mapWindow(
+				reduceFunction);
 	}
 
 	/**
@@ -289,7 +335,8 @@ public class WindowedDataStream<OUT> {
 	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction,
 			TypeInformation<R> outType) {
 
-		return discretize(true).mapWindow(reduceFunction, outType);
+		return discretize(WindowTransformation.MAPWINDOW.with(reduceFunction)).mapWindow(
+				reduceFunction, outType);
 	}
 
 	public DataStream<OUT> flatten() {
@@ -533,7 +580,7 @@ public class WindowedDataStream<OUT> {
 
 	}
 
-	protected EvictionPolicy<OUT> getEvicter() {
+	protected EvictionPolicy<OUT> getEviction() {
 
 		if (evictionHelper != null) {
 			return evictionHelper.toEvict();
@@ -571,7 +618,7 @@ public class WindowedDataStream<OUT> {
 	}
 
 	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
-		return discretize(true).getDiscretizedStream();
+		return discretize(WindowTransformation.NONE).getDiscretizedStream();
 	}
 
 	protected static class WindowKey<R> implements KeySelector<StreamWindow<R>, Integer> {

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
index e1a56cc..fe6c41a 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/StreamReduceInvokable.java
@@ -51,7 +51,7 @@ public class StreamReduceInvokable<IN> extends ChainableInvokable<IN, IN> {
 		nextValue = nextObject;
 
 		if (currentValue != null) {
-			currentValue = reducer.reduce(currentValue, nextValue);
+			currentValue = reducer.reduce(copy(currentValue), nextValue);
 		} else {
 			currentValue = nextValue;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java
new file mode 100644
index 0000000..4c6e7cd
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBuffer.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.util.LinkedList;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.util.Collector;
+
+public class BasicWindowBuffer<T> implements WindowBuffer<T> {
+
+	private static final long serialVersionUID = 1L;
+	protected LinkedList<T> buffer;
+
+	public BasicWindowBuffer() {
+		this.buffer = new LinkedList<T>();
+	}
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
+		if (!buffer.isEmpty()) {
+			StreamWindow<T> currentWindow = new StreamWindow<T>();
+			currentWindow.addAll(buffer);
+			collector.collect(currentWindow);
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	public void store(T element) throws Exception {
+		buffer.add(element);
+	}
+
+	public void evict(int n) {
+		for (int i = 0; i < n; i++) {
+			try {
+				buffer.removeFirst();
+			} catch (NoSuchElementException e) {
+				// In case no more elements are in the buffer:
+				// Prevent failure and stop deleting.
+				break;
+			}
+		}
+	}
+
+	public int size() {
+		return buffer.size();
+	}
+
+	@Override
+	public BasicWindowBuffer<T> clone() {
+		return new BasicWindowBuffer<T>();
+	}
+
+	@Override
+	public String toString() {
+		return buffer.toString();
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
index e90726d..ae6a2d9 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
@@ -37,11 +37,13 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 	protected Configuration parameters;
 	protected CloneableTriggerPolicy<IN> triggerPolicy;
 	protected CloneableEvictionPolicy<IN> evictionPolicy;
+	protected WindowBuffer<IN> windowBuffer;
 
 	protected Map<Object, StreamDiscretizer<IN>> groupedDiscretizers;
 
 	public GroupedStreamDiscretizer(KeySelector<IN, ?> keySelector,
-			CloneableTriggerPolicy<IN> triggerPolicy, CloneableEvictionPolicy<IN> evictionPolicy) {
+			CloneableTriggerPolicy<IN> triggerPolicy, CloneableEvictionPolicy<IN> evictionPolicy,
+			WindowBuffer<IN> windowBuffer) {
 
 		super(null);
 
@@ -51,6 +53,7 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 		this.evictionPolicy = evictionPolicy;
 
 		this.groupedDiscretizers = new HashMap<Object, StreamDiscretizer<IN>>();
+		this.windowBuffer = windowBuffer;
 
 	}
 
@@ -76,9 +79,8 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 			readNext();
 		}
 
-		// finally trigger the buffer.
 		for (StreamDiscretizer<IN> group : groupedDiscretizers.values()) {
-			group.emitFinalWindow();
+			group.emitWindow();
 		}
 
 	}
@@ -95,7 +97,7 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 	protected StreamDiscretizer<IN> makeNewGroup(Object key) throws Exception {
 
 		StreamDiscretizer<IN> groupDiscretizer = new StreamDiscretizer<IN>(triggerPolicy.clone(),
-				evictionPolicy.clone());
+				evictionPolicy.clone(), windowBuffer.clone());
 
 		groupDiscretizer.collector = taskContext.getOutputCollector();
 		groupDiscretizer.open(this.parameters);

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
index 63901a8..6d38ed9 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
@@ -17,10 +17,6 @@
 
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
@@ -33,9 +29,10 @@ public class GroupedTimeDiscretizer<IN> extends GroupedStreamDiscretizer<IN> {
 	private Thread policyThread;
 
 	public GroupedTimeDiscretizer(KeySelector<IN, ?> keySelector,
-			TimeTriggerPolicy<IN> triggerPolicy, CloneableEvictionPolicy<IN> evictionPolicy) {
+			TimeTriggerPolicy<IN> triggerPolicy, CloneableEvictionPolicy<IN> evictionPolicy,
+			WindowBuffer<IN> windowBuffer) {
 
-		super(keySelector, triggerPolicy, evictionPolicy);
+		super(keySelector, triggerPolicy, evictionPolicy, windowBuffer);
 		this.timeTriggerPolicy = triggerPolicy;
 	}
 
@@ -43,7 +40,7 @@ public class GroupedTimeDiscretizer<IN> extends GroupedStreamDiscretizer<IN> {
 	protected StreamDiscretizer<IN> makeNewGroup(Object key) throws Exception {
 
 		StreamDiscretizer<IN> groupDiscretizer = new StreamDiscretizer<IN>(triggerPolicy.clone(),
-				evictionPolicy.clone());
+				evictionPolicy.clone(), windowBuffer.clone());
 
 		groupDiscretizer.collector = taskContext.getOutputCollector();
 		// We omit the groupDiscretizer.open(...) call here to avoid starting
@@ -60,20 +57,6 @@ public class GroupedTimeDiscretizer<IN> extends GroupedStreamDiscretizer<IN> {
 		policyThread.start();
 	}
 
-	private void removeUnusedGroups(int threshold) {
-		List<Object> toRemove = new ArrayList<Object>();
-
-		for (Entry<Object, StreamDiscretizer<IN>> entry : groupedDiscretizers.entrySet()) {
-			if (entry.getValue().emptyCount > threshold) {
-				toRemove.add(entry.getKey());
-			}
-		}
-
-		for (Object key : toRemove) {
-			groupedDiscretizers.remove(key);
-		}
-	}
-
 	private class TimeCheck implements Runnable {
 
 		@Override
@@ -93,8 +76,6 @@ public class GroupedTimeDiscretizer<IN> extends GroupedStreamDiscretizer<IN> {
 						group.triggerOnFakeElement(fake);
 					}
 				}
-
-				removeUnusedGroups(10);
 			}
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
index ac02af3..6b0bcec 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
@@ -17,9 +17,6 @@
 
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
-import java.util.LinkedList;
-import java.util.NoSuchElementException;
-
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
@@ -39,10 +36,12 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 	private boolean isActiveTrigger;
 	private boolean isActiveEviction;
 	private Thread activePolicyThread;
-	protected LinkedList<IN> buffer;
 	public int emptyCount = 0;
 
-	public StreamDiscretizer(TriggerPolicy<IN> triggerPolicy, EvictionPolicy<IN> evictionPolicy) {
+	protected WindowBuffer<IN> bufferHandler;
+
+	public StreamDiscretizer(TriggerPolicy<IN> triggerPolicy, EvictionPolicy<IN> evictionPolicy,
+			WindowBuffer<IN> bufferHandler) {
 		super(null);
 
 		this.triggerPolicy = triggerPolicy;
@@ -51,7 +50,7 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 		this.isActiveTrigger = triggerPolicy instanceof ActiveTriggerPolicy;
 		this.isActiveEviction = evictionPolicy instanceof ActiveEvictionPolicy;
 
-		this.buffer = new LinkedList<IN>();
+		this.bufferHandler = bufferHandler;
 	}
 
 	@Override
@@ -66,7 +65,7 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 			activePolicyThread.interrupt();
 		}
 
-		emitFinalWindow();
+		emitWindow();
 
 	}
 
@@ -77,8 +76,9 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 	 * 
 	 * @param input
 	 *            a real input element
+	 * @throws Exception
 	 */
-	protected synchronized void processRealElement(IN input) {
+	protected synchronized void processRealElement(IN input) throws Exception {
 
 		if (isActiveTrigger) {
 			ActiveTriggerPolicy<IN> trigger = (ActiveTriggerPolicy<IN>) triggerPolicy;
@@ -97,7 +97,7 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 
 		evict(input, isTriggered);
 
-		buffer.add(input);
+		bufferHandler.store(input);
 
 	}
 
@@ -124,10 +124,7 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 	 * if not empty
 	 */
 	protected void emitWindow() {
-		if (!buffer.isEmpty()) {
-			StreamWindow<IN> currentWindow = new StreamWindow<IN>();
-			currentWindow.addAll(buffer);
-			collector.collect(currentWindow);
+		if (bufferHandler.emitWindow(collector)) {
 			emptyCount = 0;
 		} else {
 			emptyCount++;
@@ -139,37 +136,16 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 
 		if (isActiveEviction) {
 			ActiveEvictionPolicy<IN> ep = (ActiveEvictionPolicy<IN>) evictionPolicy;
-			numToEvict = ep.notifyEvictionWithFakeElement(input, buffer.size());
+			numToEvict = ep.notifyEvictionWithFakeElement(input, bufferHandler.size());
 		}
 
-		evictFromBuffer(numToEvict);
+		bufferHandler.evict(numToEvict);
 	}
 
 	private void evict(IN input, boolean isTriggered) {
-		int numToEvict = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
+		int numToEvict = evictionPolicy.notifyEviction(input, isTriggered, bufferHandler.size());
 
-		evictFromBuffer(numToEvict);
-	}
-
-	private void evictFromBuffer(int n) {
-		for (int i = 0; i < n; i++) {
-			try {
-				buffer.removeFirst();
-			} catch (NoSuchElementException e) {
-				// In case no more elements are in the buffer:
-				// Prevent failure and stop deleting.
-				break;
-			}
-		}
-	}
-
-	/**
-	 * This function emits the partial windows at the end of the stream
-	 */
-	protected void emitFinalWindow() {
-		if (!buffer.isEmpty()) {
-			emitWindow();
-		}
+		bufferHandler.evict(numToEvict);
 	}
 
 	@Override
@@ -189,7 +165,7 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 
 	@Override
 	public String toString() {
-		return buffer.toString();
+		return bufferHandler.toString();
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
index 70c608c..12e3574 100755
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowSerializer.java
@@ -47,11 +47,6 @@ public final class StreamWindowSerializer<T> extends TypeSerializer<StreamWindow
 	}
 
 	@Override
-	public boolean isStateful() {
-		return false;
-	}
-
-	@Override
 	public StreamWindow<T> createInstance() {
 		return new StreamWindow<T>(0, 0, 0);
 	}
@@ -133,4 +128,9 @@ public final class StreamWindowSerializer<T> extends TypeSerializer<StreamWindow
 	public void copy(DataInputView source, DataOutputView target) throws IOException {
 		// Needs to be implemented
 	}
+
+	@Override
+	public TypeSerializer<StreamWindow<T>> duplicate() {
+		return this;
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
new file mode 100644
index 0000000..4f05813
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.util.Collector;
+
+public class TumblingPreReducer<T> implements WindowBuffer<T> {
+
+	private static final long serialVersionUID = 1L;
+
+	private ReduceFunction<T> reducer;
+
+	private T reduced;
+	private int numOfElements = 0;
+	private TypeSerializer<T> serializer;
+
+	public TumblingPreReducer(ReduceFunction<T> reducer, TypeSerializer<T> serializer) {
+		this.reducer = reducer;
+		this.serializer = serializer;
+	}
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
+		if (reduced != null) {
+			StreamWindow<T> currentWindow = new StreamWindow<T>();
+			currentWindow.add(reduced);
+			collector.collect(currentWindow);
+			reduced = null;
+			numOfElements = 0;
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	public void store(T element) throws Exception {
+		if (reduced == null) {
+			reduced = element;
+		} else {
+			reduced = reducer.reduce(serializer.copy(reduced), element);
+		}
+		numOfElements++;
+	}
+
+	public void evict(int n) {
+	}
+
+	public int size() {
+		return numOfElements;
+	}
+
+	@Override
+	public TumblingPreReducer<T> clone() {
+		return new TumblingPreReducer<T>(reducer, serializer);
+	}
+
+	@Override
+	public String toString() {
+		return reduced.toString();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java
new file mode 100644
index 0000000..ef8fc2b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowBuffer.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.io.Serializable;
+
+import org.apache.flink.util.Collector;
+
+public interface WindowBuffer<T> extends Serializable {
+
+	public void store(T element) throws Exception;
+
+	public void evict(int n);
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector);
+
+	public int size();
+	
+	public WindowBuffer<T> clone();
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
index 8a6a7ce..04f09e0 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
@@ -82,7 +82,7 @@ public class GroupedStreamDiscretizerTest {
 		CloneableEvictionPolicy<Integer> eviction = new TumblingEvictionPolicy<Integer>();
 
 		GroupedStreamDiscretizer<Integer> discretizer = new GroupedStreamDiscretizer<Integer>(
-				keySelector, trigger, eviction);
+				keySelector, trigger, eviction, new BasicWindowBuffer<Integer>());
 
 		List<StreamWindow<Integer>> result = MockContext.createAndExecute(discretizer, inputs);
 		assertEquals(expected, new HashSet<StreamWindow<Integer>>(result));

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
index 8fcac8c..706b0ef 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
@@ -74,7 +74,8 @@ public class StreamDiscretizerTest {
 		EvictionPolicy<Integer> eviction = new TimeEvictionPolicy<Integer>(4L,
 				new TimestampWrapper<Integer>(myTimeStamp, 1));
 
-		StreamDiscretizer<Integer> discretizer = new StreamDiscretizer<Integer>(trigger, eviction);
+		StreamDiscretizer<Integer> discretizer = new StreamDiscretizer<Integer>(trigger, eviction,
+				new BasicWindowBuffer<Integer>());
 
 		List<StreamWindow<Integer>> result = MockContext.createAndExecute(discretizer, inputs);
 		assertEquals(expected, result);
@@ -99,7 +100,8 @@ public class StreamDiscretizerTest {
 
 		EvictionPolicy<Integer> eviction = new TumblingEvictionPolicy<Integer>();
 
-		StreamDiscretizer<Integer> discretizer = new StreamDiscretizer<Integer>(trigger, eviction);
+		StreamDiscretizer<Integer> discretizer = new StreamDiscretizer<Integer>(trigger, eviction,
+				new BasicWindowBuffer<Integer>());
 
 		List<StreamWindow<Integer>> result = MockContext.createAndExecute(discretizer, inputs);
 		assertEquals(expected, result);

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
index c1687e6..bcde0c5 100755
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowIntegrationTest.java
@@ -101,6 +101,8 @@ public class WindowIntegrationTest implements Serializable {
 		source.groupBy(new ModKey(3)).window(Count.of(2)).groupBy(new ModKey(2))
 				.mapWindow(new IdentityWindowMap()).flatten().addSink(new DistributedSink2());
 
+		env.generateSequence(1, 10).window(Count.of(3)).sum(0).getDiscretizedStream().print();
+		
 		env.execute();
 
 		// sum ( Count of 2 slide 3 )

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
index ce5db4a..af10e52 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/StockPrices.java
@@ -78,6 +78,7 @@ public class StockPrices {
 	// PROGRAM
 	// *************************************************************************
 
+	@SuppressWarnings({ "serial", "unused" })
 	public static void main(String[] args) throws Exception {
 
 		if (!parseParameters(args)) {
@@ -108,6 +109,7 @@ public class StockPrices {
 		DataStream<StockPrice> BUX_stream = env.addSource(new StockSource("BUX", 40));
 
 		//Merge all stock streams together
+		@SuppressWarnings("unchecked")
 		DataStream<StockPrice> stockStream = socketStockStream.merge(SPX_stream, FTSE_stream, DJI_stream, BUX_stream);
 		
 		//Step 2
@@ -116,9 +118,9 @@ public class StockPrices {
 				.window(Time.of(10, TimeUnit.SECONDS))
 				.every(Time.of(5, TimeUnit.SECONDS));
 
-		DataStream<StockPrice> lowest = windowedStream.minBy("price").setParallelism(1);
-		DataStream<StockPrice> maxByStock = windowedStream.groupBy("symbol").maxBy("price");
-		DataStream<StockPrice> rollingMean = windowedStream.groupBy("symbol").reduceGroup(new MeanReduce());
+		DataStream<StockPrice> lowest = windowedStream.minBy("price").flatten();
+		DataStream<StockPrice> maxByStock = windowedStream.groupBy("symbol").maxBy("price").flatten();
+		DataStream<StockPrice> rollingMean = windowedStream.groupBy("symbol").mapWindow(new MeanReduce()).flatten();
 
 		//Step 3
 		//Use  delta policy to create price change warnings, and also count the number of warning every half minute
@@ -130,7 +132,7 @@ public class StockPrices {
 						return Math.abs(oldDataPoint.price - newDataPoint.price);
 					}
 				}, DEFAULT_STOCK_PRICE))
-				.reduceGroup(new SendWarning());
+				.mapWindow(new SendWarning()).flatten();
 
 
 		DataStream<Count> warningsPerStock = priceWarnings.map(new MapFunction<String, Count>() {
@@ -138,7 +140,7 @@ public class StockPrices {
 			public Count map(String value) throws Exception {
 				return new Count(value, 1);
 			}
-		}).groupBy("symbol").window(Time.of(30, TimeUnit.SECONDS)).sum("count");
+		}).groupBy("symbol").window(Time.of(30, TimeUnit.SECONDS)).sum("count").flatten();
 
 		//Step 4
 		//Read a stream of tweets and extract the stock symbols
@@ -166,7 +168,7 @@ public class StockPrices {
 			}
 		}).groupBy("symbol")
 				.window(Time.of(30, TimeUnit.SECONDS))
-				.sum("count");
+				.sum("count").flatten();
 
 		//Step 5
 		//For advanced analysis we join the number of tweets and the number of price change warnings by stock
@@ -186,8 +188,7 @@ public class StockPrices {
 
 		DataStream<Double> rollingCorrelation = tweetsAndWarning
 				.window(Time.of(30, TimeUnit.SECONDS))
-				.reduceGroup(new CorrelationReduce())
-				.setParallelism(1);
+				.mapWindow(new CorrelationReduce()).flatten();
 
 		if (fileOutput) {
 			rollingCorrelation.writeAsText(outputPath, 1);
@@ -205,6 +206,7 @@ public class StockPrices {
 
 	public static class StockPrice implements Serializable {
 
+		private static final long serialVersionUID = 1L;
 		public String symbol;
 		public Double price;
 
@@ -226,6 +228,8 @@ public class StockPrices {
 	}
 
 	public static class Count implements Serializable {
+		
+		private static final long serialVersionUID = 1L;
 		public String symbol;
 		public Integer count;
 
@@ -252,6 +256,7 @@ public class StockPrices {
 
 	public final static class StockSource implements SourceFunction<StockPrice> {
 
+		private static final long serialVersionUID = 1L;
 		private Double price;
 		private String symbol;
 		private Integer sigma;
@@ -276,6 +281,7 @@ public class StockPrices {
 
 	public final static class MeanReduce implements GroupReduceFunction<StockPrice, StockPrice> {
 
+		private static final long serialVersionUID = 1L;
 		private Double sum = 0.0;
 		private Integer count = 0;
 		private String symbol = "";
@@ -296,6 +302,7 @@ public class StockPrices {
 
 	public static final class TweetSource implements SourceFunction<String> {
 
+		private static final long serialVersionUID = 1L;
 		Random random;
 		StringBuilder stringBuilder;
 
@@ -318,6 +325,9 @@ public class StockPrices {
 	}
 
 	public static final class SendWarning implements GroupReduceFunction<StockPrice, String> {
+
+		private static final long serialVersionUID = 1L;
+
 		@Override
 		public void reduce(Iterable<StockPrice> values, Collector<String> out) throws Exception {
 			if (values.iterator().hasNext()) {
@@ -328,6 +338,7 @@ public class StockPrices {
 
 	public static final class CorrelationReduce implements GroupReduceFunction<Tuple2<Integer, Integer>, Double> {
 
+		private static final long serialVersionUID = 1L;
 		private Integer leftSum;
 		private Integer rightSum;
 		private Integer count;

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/StockPrices.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/StockPrices.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/StockPrices.scala
index 8a0ce5e..222a0a7 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/StockPrices.scala
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/StockPrices.scala
@@ -80,7 +80,7 @@ object StockPrices {
     //Read a stream of stock prices from different sources and merge it into one stream
 
     //Read from a socket stream at map it to StockPrice objects
-    val socketStockStream = env.socketTextStream("localhost", 9999).map(x => {
+    val socketStockStream = env.socketTextStream(hostName, port).map(x => {
       val split = x.split(",")
       StockPrice(split(0), split(1).toDouble)
     })
@@ -98,9 +98,9 @@ object StockPrices {
     //Compute some simple statistics on a rolling window
     val windowedStream = stockStream.window(Time.of(10, SECONDS)).every(Time.of(5, SECONDS))
 
-    val lowest = windowedStream.minBy("price").setParallelism(1)
-    val maxByStock = windowedStream.groupBy("symbol").maxBy("price")
-    val rollingMean = windowedStream.groupBy("symbol").reduceGroup(mean _)
+    val lowest = windowedStream.minBy("price")
+    val maxByStock = windowedStream.groupBy("symbol").maxBy("price").getDiscretizedStream
+    val rollingMean = windowedStream.groupBy("symbol").mapWindow(mean _).getDiscretizedStream
 
     //Step 3 
     //Use  delta policy to create price change warnings,
@@ -108,13 +108,13 @@ object StockPrices {
 
     val priceWarnings = stockStream.groupBy("symbol")
       .window(Delta.of(0.05, priceChange, defaultPrice))
-      .reduceGroup(sendWarning _)
-
+      .mapWindow(sendWarning _)
+      
     val warningsPerStock = priceWarnings.map(Count(_, 1))
-      .groupBy("symbol")
       .window(Time.of(30, SECONDS))
+      .groupBy("symbol")
       .sum("count")
-
+      
     //Step 4 
     //Read a stream of tweets and extract the stock symbols
 
@@ -125,10 +125,10 @@ object StockPrices {
       .filter(symbols.contains(_))                     
                     
     val tweetsPerStock = mentionedSymbols.map(Count(_, 1))
-      .groupBy("symbol")
       .window(Time.of(30, SECONDS))
+      .groupBy("symbol")
       .sum("count")
-
+      
     //Step 5
     //For advanced analysis we join the number of tweets and
     //the number of price change warnings by stock
@@ -140,11 +140,16 @@ object StockPrices {
       .onWindow(30, SECONDS)
       .where("symbol")
       .equalTo("symbol") { (c1, c2) => (c1.count, c2.count) }
+    
 
     val rollingCorrelation = tweetsAndWarning.window(Time.of(30, SECONDS))
-      .reduceGroup(computeCorrelation _).setParallelism(1)
+      .mapWindow(computeCorrelation _)
 
-    rollingCorrelation.print
+    if (fileOutput) {
+      rollingCorrelation.writeAsText(outputPath, 1);
+    } else {
+      rollingCorrelation.print;
+    }
 
     env.execute("Stock stream")
   }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef7b7cd9/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
index 61d7db1..cdda8f2 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/package.scala
@@ -26,8 +26,8 @@ import org.apache.flink.streaming.api.datastream.{ DataStream => JavaStream }
 import org.apache.flink.streaming.api.datastream.{ WindowedDataStream => JavaWStream }
 import org.apache.flink.streaming.api.datastream.{ SplitDataStream => SplitJavaStream }
 import org.apache.flink.streaming.api.datastream.{ ConnectedDataStream => JavaConStream }
-
 import language.implicitConversions
+import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow
 
 package object scala {
   // We have this here so that we always have generated TypeInformationS when
@@ -48,7 +48,9 @@ package object scala {
 
   implicit def seqToFlinkSource[T: ClassTag: TypeInformation](scalaSeq: Seq[T]) : DataStream[T] =
     StreamExecutionEnvironment.getExecutionEnvironment.fromCollection(scalaSeq)
-
+    
+  implicit def windowedToDataStream[R](windowedStream: WindowedDataStream[R]): DataStream[R] =
+    windowedStream.flatten      
 
   private[flink] def fieldNames2Indices(
       typeInfo: TypeInformation[_],


[02/20] flink git commit: [streaming] StreamDiscretizer rework to support only 1 eviction and trigger for robustness + test cleanup

Posted by mb...@apache.org.
[streaming] StreamDiscretizer rework to support only 1 eviction and trigger for robustness + test cleanup


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/412779fa
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/412779fa
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/412779fa

Branch: refs/heads/master
Commit: 412779fae5b20513997fcacee509082d3fa78f96
Parents: c560d76
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Feb 8 13:09:34 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:07 2015 +0100

----------------------------------------------------------------------
 .../streaming/api/datastream/DataStream.java    |   7 +-
 .../api/datastream/WindowedDataStream.java      | 180 ++----
 .../api/invokable/StreamInvokable.java          |   2 +-
 .../windowing/GroupedStreamDiscretizer.java     | 437 +-------------
 .../operator/windowing/StreamDiscretizer.java   | 363 +++---------
 .../operator/windowing/StreamWindow.java        |  15 +-
 .../operator/windowing/WindowFlattener.java     |   1 +
 .../apache/flink/streaming/api/PrintTest.java   |  57 --
 .../windowing/GroupedStreamDiscretizerTest.java |  92 +++
 .../windowing/GroupedWindowInvokableTest.java   | 574 -------------------
 .../windowing/StreamDiscretizerTest.java        | 107 ++++
 .../operator/windowing/StreamWindowTest.java    |  14 +-
 .../operator/windowing/WindowInvokableTest.java | 261 ---------
 13 files changed, 382 insertions(+), 1728 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index eff6026..b3d0564 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -865,7 +865,7 @@ public class DataStream<OUT> {
 	 * @return A {@link WindowedDataStream} providing further operations.
 	 */
 	@SuppressWarnings({ "rawtypes", "unchecked" })
-	public WindowedDataStream<OUT> window(WindowingHelper... policyHelpers) {
+	public WindowedDataStream<OUT> window(WindowingHelper policyHelpers) {
 		return new WindowedDataStream<OUT>(this, policyHelpers);
 	}
 
@@ -884,9 +884,8 @@ public class DataStream<OUT> {
 	 *            number of elements in each time window.
 	 * @return A {@link WindowedDataStream} providing further operations.
 	 */
-	public WindowedDataStream<OUT> window(List<TriggerPolicy<OUT>> triggers,
-			List<EvictionPolicy<OUT>> evicters) {
-		return new WindowedDataStream<OUT>(this, triggers, evicters);
+	public WindowedDataStream<OUT> window(TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> evicter) {
+		return new WindowedDataStream<OUT>(this, trigger, evicter);
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index fc117a1..e2cb3f7 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -17,10 +17,6 @@
 
 package org.apache.flink.streaming.api.datastream;
 
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
 import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
@@ -44,7 +40,6 @@ import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
 import org.apache.flink.streaming.util.keys.KeySelectorUtil;
@@ -64,54 +59,34 @@ public class WindowedDataStream<OUT> {
 	protected DataStream<OUT> dataStream;
 
 	protected boolean isLocal = false;
-	protected boolean isCentral = true;
 
 	protected KeySelector<OUT, ?> discretizerKey;
 	protected KeySelector<OUT, ?> groupByKey;
 
-	protected List<WindowingHelper<OUT>> triggerHelpers;
-	protected List<WindowingHelper<OUT>> evictionHelpers;
-
-	protected LinkedList<TriggerPolicy<OUT>> userTriggers;
-	protected LinkedList<EvictionPolicy<OUT>> userEvicters;
-
-	protected WindowedDataStream() {
+	protected WindowingHelper<OUT> triggerHelper;
+	protected WindowingHelper<OUT> evictionHelper;
 
-	}
+	protected TriggerPolicy<OUT> userTrigger;
+	protected EvictionPolicy<OUT> userEvicter;
 
-	protected WindowedDataStream(DataStream<OUT> dataStream, WindowingHelper<OUT>... policyHelpers) {
+	protected WindowedDataStream(DataStream<OUT> dataStream, WindowingHelper<OUT> policyHelper) {
 		this.dataStream = dataStream.copy();
-		this.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
-		for (WindowingHelper<OUT> helper : policyHelpers) {
-			this.triggerHelpers.add(helper);
-		}
+		this.triggerHelper = policyHelper;
 
 		if (dataStream instanceof GroupedDataStream) {
 			this.discretizerKey = ((GroupedDataStream<OUT>) dataStream).keySelector;
-			// set all policies distributed
-			this.isCentral = false;
 		}
 	}
 
-	protected WindowedDataStream(DataStream<OUT> dataStream, List<TriggerPolicy<OUT>> triggers,
-			List<EvictionPolicy<OUT>> evicters) {
+	protected WindowedDataStream(DataStream<OUT> dataStream, TriggerPolicy<OUT> trigger,
+			EvictionPolicy<OUT> evicter) {
 		this.dataStream = dataStream.copy();
 
-		if (triggers != null) {
-			this.userTriggers = new LinkedList<TriggerPolicy<OUT>>();
-			this.userTriggers.addAll(triggers);
-		}
-
-		if (evicters != null) {
-			this.userEvicters = new LinkedList<EvictionPolicy<OUT>>();
-			this.userEvicters.addAll(evicters);
-		}
+		this.userTrigger = trigger;
+		this.userEvicter = evicter;
 
 		if (dataStream instanceof GroupedDataStream) {
 			this.discretizerKey = ((GroupedDataStream<OUT>) dataStream).keySelector;
-			// set all policies distributed
-			this.isCentral = false;
-
 		}
 	}
 
@@ -119,14 +94,16 @@ public class WindowedDataStream<OUT> {
 		this.dataStream = windowedDataStream.dataStream.copy();
 		this.discretizerKey = windowedDataStream.discretizerKey;
 		this.groupByKey = windowedDataStream.groupByKey;
-		this.triggerHelpers = windowedDataStream.triggerHelpers;
-		this.evictionHelpers = windowedDataStream.evictionHelpers;
-		this.userTriggers = windowedDataStream.userTriggers;
-		this.userEvicters = windowedDataStream.userEvicters;
-		this.isCentral = windowedDataStream.isCentral;
+		this.triggerHelper = windowedDataStream.triggerHelper;
+		this.evictionHelper = windowedDataStream.evictionHelper;
+		this.userTrigger = windowedDataStream.userTrigger;
+		this.userEvicter = windowedDataStream.userEvicter;
 		this.isLocal = windowedDataStream.isLocal;
 	}
 
+	public WindowedDataStream() {
+	}
+
 	public <F> F clean(F f) {
 		return dataStream.clean(f);
 	}
@@ -146,15 +123,13 @@ public class WindowedDataStream<OUT> {
 	 * @return The windowed data stream with triggering set
 	 */
 	@SuppressWarnings({ "unchecked", "rawtypes" })
-	public WindowedDataStream<OUT> every(WindowingHelper... policyHelpers) {
+	public WindowedDataStream<OUT> every(WindowingHelper policyHelper) {
 		WindowedDataStream<OUT> ret = this.copy();
-		if (ret.evictionHelpers == null) {
-			ret.evictionHelpers = ret.triggerHelpers;
-			ret.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
-		}
-		for (WindowingHelper<OUT> helper : policyHelpers) {
-			ret.triggerHelpers.add(helper);
+		if (ret.evictionHelper == null) {
+			ret.evictionHelper = ret.triggerHelper;
+			ret.triggerHelper = policyHelper;
 		}
+
 		return ret;
 	}
 
@@ -238,11 +213,11 @@ public class WindowedDataStream<OUT> {
 		StreamInvokable<OUT, StreamWindow<OUT>> discretizer;
 
 		if (discretizerKey == null) {
-			discretizer = new StreamDiscretizer<OUT>(getTriggers(), getEvicters());
+			discretizer = new StreamDiscretizer<OUT>(getTrigger(), getEvicter());
 		} else {
 			discretizer = new GroupedStreamDiscretizer<OUT>(discretizerKey,
-					getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
-					getCentralEvicters());
+					(CloneableTriggerPolicy<OUT>) getTrigger(),
+					(CloneableEvictionPolicy<OUT>) getEvicter());
 		}
 
 		int parallelism = isLocal || (discretizerKey != null) ? dataStream.environment
@@ -537,103 +512,32 @@ public class WindowedDataStream<OUT> {
 		return reduceWindow(aggregator);
 	}
 
-	protected LinkedList<TriggerPolicy<OUT>> getTriggers() {
-
-		LinkedList<TriggerPolicy<OUT>> triggers = new LinkedList<TriggerPolicy<OUT>>();
-
-		if (triggerHelpers != null) {
-			for (WindowingHelper<OUT> helper : triggerHelpers) {
-				triggers.add(helper.toTrigger());
-			}
-		}
-
-		if (userTriggers != null) {
-			triggers.addAll(userTriggers);
-		}
-
-		return triggers;
-
-	}
-
-	protected LinkedList<EvictionPolicy<OUT>> getEvicters() {
+	protected TriggerPolicy<OUT> getTrigger() {
 
-		LinkedList<EvictionPolicy<OUT>> evicters = new LinkedList<EvictionPolicy<OUT>>();
-
-		if (evictionHelpers != null) {
-			for (WindowingHelper<OUT> helper : evictionHelpers) {
-				evicters.add(helper.toEvict());
-			}
+		if (triggerHelper != null) {
+			return triggerHelper.toTrigger();
+		} else if (userTrigger != null) {
+			return userTrigger;
 		} else {
-			if (userEvicters == null) {
-				boolean notOnlyTime = false;
-				for (WindowingHelper<OUT> helper : triggerHelpers) {
-					if (helper instanceof Time<?>) {
-						evicters.add(helper.toEvict());
-					} else {
-						notOnlyTime = true;
-					}
-				}
-				if (notOnlyTime) {
-					evicters.add(new TumblingEvictionPolicy<OUT>());
-				}
-			}
-		}
-
-		if (userEvicters != null) {
-			evicters.addAll(userEvicters);
-		}
-
-		return evicters;
-	}
-
-	protected LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
-		LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
-		if (isCentral) {
-			cTriggers.addAll(getTriggers());
-		} else {
-			for (TriggerPolicy<OUT> trigger : getTriggers()) {
-				if (trigger instanceof TimeTriggerPolicy) {
-					cTriggers.add(trigger);
-				}
-			}
-		}
-		return cTriggers;
-	}
-
-	protected LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
-		LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = null;
-
-		if (!isCentral) {
-			dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
-			for (TriggerPolicy<OUT> trigger : getTriggers()) {
-				if (!(trigger instanceof TimeTriggerPolicy)) {
-					dTriggers.add((CloneableTriggerPolicy<OUT>) trigger);
-				}
-			}
+			throw new RuntimeException("Trigger must not be null");
 		}
 
-		return dTriggers;
 	}
 
-	protected LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
-		LinkedList<CloneableEvictionPolicy<OUT>> evicters = null;
+	protected EvictionPolicy<OUT> getEvicter() {
 
-		if (!isCentral) {
-			evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
-			for (EvictionPolicy<OUT> evicter : getEvicters()) {
-				evicters.add((CloneableEvictionPolicy<OUT>) evicter);
+		if (evictionHelper != null) {
+			return evictionHelper.toEvict();
+		} else if (userEvicter == null) {
+			if (triggerHelper instanceof Time) {
+				return triggerHelper.toEvict();
+			} else {
+				return new TumblingEvictionPolicy<OUT>();
 			}
-		}
-
-		return evicters;
-	}
-
-	protected LinkedList<EvictionPolicy<OUT>> getCentralEvicters() {
-		if (isCentral) {
-			return getEvicters();
 		} else {
-			return null;
+			return userEvicter;
 		}
+
 	}
 
 	/**
@@ -664,7 +568,7 @@ public class WindowedDataStream<OUT> {
 	protected static class WindowKey<R> implements KeySelector<StreamWindow<R>, Integer> {
 
 		private static final long serialVersionUID = 1L;
-		
+
 		@Override
 		public Integer getKey(StreamWindow<R> value) throws Exception {
 			return value.windowID;

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
index f6e7052..7feeac8 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/StreamInvokable.java
@@ -58,7 +58,7 @@ public abstract class StreamInvokable<IN, OUT> implements Serializable {
 	protected IN nextObject;
 	protected boolean isMutable;
 
-	protected Collector<OUT> collector;
+	public Collector<OUT> collector;
 	protected Function userFunction;
 	protected volatile boolean isRunning;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
index 6178f48..efd2e06 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
@@ -18,57 +18,14 @@
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.Map;
-import java.util.NoSuchElementException;
 
-import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
 
-/**
- * This invokable allows windowing based on {@link TriggerPolicy} and
- * {@link EvictionPolicy} instances including their active and cloneable
- * versions. It is additionally aware of the creation of windows per group.
- * 
- * A {@link KeySelector} is used to specify the key position or key extraction.
- * The {@link ReduceFunction} will be executed on each group separately.
- * Policies might either be centralized or distributed. It is not possible to
- * use central and distributed eviction policies at the same time. A distributed
- * policy have to be a {@link CloneableTriggerPolicy} or
- * {@link CloneableEvictionPolicy} as it will be cloned to have separated
- * instances for each group. At the startup time the distributed policies will
- * be stored as sample, and only clones of them will be used to maintain the
- * groups. Therefore, each group starts with the initial policy states.
- * 
- * While a distributed policy only gets notified with the elements belonging to
- * the respective group, a centralized policy get notified with all arriving
- * elements. When a centralized trigger occurred, all groups get triggered. This
- * is done by submitting the element which caused the trigger as real element to
- * the groups it belongs to and as fake element to all other groups. Within the
- * groups the element might be further processed, causing more triggers,
- * prenotifications of active distributed policies and evictions like usual.
- * 
- * Central policies can be instance of {@link ActiveTriggerPolicy} and also
- * implement the
- * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
- * method. Fake elements created on prenotification will be forwarded to all
- * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
- * it forwards/distributed calls all groups.
- * 
- * @param <IN>
- *            The type of input elements handled by this operator invokable.
- */
 public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>> {
 
 	/**
@@ -78,239 +35,50 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 
 	private KeySelector<IN, ?> keySelector;
 	private Configuration parameters;
-	private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies;
-	private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies;
-	private LinkedList<ActiveEvictionPolicy<IN>> activeCentralEvictionPolicies;
-	private LinkedList<EvictionPolicy<IN>> centralEvictionPolicies;
-	private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies;
-	private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies;
-	private Map<Object, StreamDiscretizer<IN>> windowingGroups;
-	private LinkedList<Thread> activePolicyThreads;
-	private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
-	private LinkedList<StreamDiscretizer<IN>> deleteOrderForCentralEviction;
+	private CloneableTriggerPolicy<IN> triggerPolicy;
+	private CloneableEvictionPolicy<IN> evictionPolicy;
+
+	private Map<Object, StreamDiscretizer<IN>> groupedDiscretizers;
 
-	/**
-	 * This constructor creates an instance of the grouped windowing invokable.
-	 * 
-	 * A {@link KeySelector} is used to specify the key position or key
-	 * extraction. The {@link ReduceFunction} will be executed on each group
-	 * separately. Policies might either be centralized or distributed. It is
-	 * not possible to use central and distributed eviction policies at the same
-	 * time. A distributed policy have to be a {@link CloneableTriggerPolicy} or
-	 * {@link CloneableEvictionPolicy} as it will be cloned to have separated
-	 * instances for each group. At the startup time the distributed policies
-	 * will be stored as sample, and only clones of them will be used to
-	 * maintain the groups. Therefore, each group starts with the initial policy
-	 * states.
-	 * 
-	 * While a distributed policy only gets notified with the elements belonging
-	 * to the respective group, a centralized policy get notified with all
-	 * arriving elements. When a centralized trigger occurred, all groups get
-	 * triggered. This is done by submitting the element which caused the
-	 * trigger as real element to the groups it belongs to and as fake element
-	 * to all other groups. Within the groups the element might be further
-	 * processed, causing more triggers, prenotifications of active distributed
-	 * policies and evictions like usual.
-	 * 
-	 * Central policies can be instance of {@link ActiveTriggerPolicy} and also
-	 * implement the
-	 * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
-	 * method. Fake elements created on prenotification will be forwarded to all
-	 * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
-	 * that it forwards/distributed calls all groups.
-	 * 
-	 * @param userFunction
-	 *            The user defined function.
-	 * @param keySelector
-	 *            A key selector to extract the key for the groups from the
-	 *            input data.
-	 * @param distributedTriggerPolicies
-	 *            Trigger policies to be distributed and maintained individually
-	 *            within each group.
-	 * @param distributedEvictionPolicies
-	 *            Eviction policies to be distributed and maintained
-	 *            individually within each group. Note that there cannot be
-	 *            both, central and distributed eviction policies at the same
-	 *            time.
-	 * @param centralTriggerPolicies
-	 *            Trigger policies which will only exist once at a central
-	 *            place. In case a central policy triggers, it will cause all
-	 *            groups to be emitted. (Remark: Empty groups cannot be emitted.
-	 *            If only one element is contained a group, this element itself
-	 *            is returned as aggregated result.)
-	 * @param centralEvictionPolicies
-	 *            Eviction which will only exist once at a central place. Note
-	 *            that there cannot be both, central and distributed eviction
-	 *            policies at the same time. The central eviction policy will
-	 *            work on an simulated element buffer containing all elements no
-	 *            matter which group they belong to.
-	 */
 	public GroupedStreamDiscretizer(KeySelector<IN, ?> keySelector,
-			LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
-			LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
-			LinkedList<TriggerPolicy<IN>> centralTriggerPolicies,
-			LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
+			CloneableTriggerPolicy<IN> triggerPolicy, CloneableEvictionPolicy<IN> evictionPolicy) {
 
 		super(null);
 
 		this.keySelector = keySelector;
 
-		// handle the triggers
-		if (centralTriggerPolicies != null) {
-			this.centralTriggerPolicies = centralTriggerPolicies;
-			this.activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
-
-			for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
-				if (trigger instanceof ActiveTriggerPolicy) {
-					this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
-				}
-			}
-		} else {
-			this.centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-		}
-
-		if (distributedTriggerPolicies != null) {
-			this.distributedTriggerPolicies = distributedTriggerPolicies;
-		} else {
-			this.distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
-		}
-
-		if (distributedEvictionPolicies != null) {
-			this.distributedEvictionPolicies = distributedEvictionPolicies;
-		} else {
-			this.distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
-		}
-
-		this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
-
-		if (centralEvictionPolicies != null) {
-			this.centralEvictionPolicies = centralEvictionPolicies;
-
-			for (EvictionPolicy<IN> eviction : centralEvictionPolicies) {
-				if (eviction instanceof ActiveEvictionPolicy) {
-					this.activeCentralEvictionPolicies.add((ActiveEvictionPolicy<IN>) eviction);
-				}
-			}
-		} else {
-			this.centralEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
-		}
-
-		this.windowingGroups = new HashMap<Object, StreamDiscretizer<IN>>();
-		this.activePolicyThreads = new LinkedList<Thread>();
-		this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-		this.deleteOrderForCentralEviction = new LinkedList<StreamDiscretizer<IN>>();
-
-		// check that not both, central and distributed eviction, is used at the
-		// same time.
-		if (!this.centralEvictionPolicies.isEmpty() && !this.distributedEvictionPolicies.isEmpty()) {
-			throw new UnsupportedOperationException(
-					"You can only use either central or distributed eviction policies but not both at the same time.");
-		}
+		this.triggerPolicy = triggerPolicy;
+		this.evictionPolicy = evictionPolicy;
 
-		// Check that there is at least one trigger and one eviction policy
-		if (this.centralEvictionPolicies.isEmpty() && this.distributedEvictionPolicies.isEmpty()) {
-			throw new UnsupportedOperationException(
-					"You have to define at least one eviction policy");
-		}
-		if (this.centralTriggerPolicies.isEmpty() && this.distributedTriggerPolicies.isEmpty()) {
-			throw new UnsupportedOperationException(
-					"You have to define at least one trigger policy");
-		}
+		this.groupedDiscretizers = new HashMap<Object, StreamDiscretizer<IN>>();
 
 	}
 
 	@Override
 	public void invoke() throws Exception {
-		// Prevent empty data streams
 		if (readNext() == null) {
 			throw new RuntimeException("DataStream must not be empty");
 		}
 
-		// Continuously run
 		while (nextRecord != null) {
-			StreamDiscretizer<IN> groupInvokable = windowingGroups.get(keySelector
-					.getKey(nextRecord.getObject()));
-			if (groupInvokable == null) {
-				groupInvokable = makeNewGroup(nextRecord);
-			}
-
-			// Run the precalls for central active triggers
-			for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
-				Object[] result = trigger.preNotifyTrigger(nextRecord.getObject());
-				for (Object in : result) {
-
-					// If central eviction is used, handle it here
-					if (!activeCentralEvictionPolicies.isEmpty()) {
-						evictElements(centralActiveEviction(in));
-					}
-
-					// process in groups
-					for (StreamDiscretizer<IN> group : windowingGroups.values()) {
-						group.processFakeElement(in, trigger);
-						checkForEmptyGroupBuffer(group);
-					}
-				}
-			}
-
-			// Process non-active central triggers
-			for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
-				if (triggerPolicy.notifyTrigger(nextRecord.getObject())) {
-					currentTriggerPolicies.add(triggerPolicy);
-				}
-			}
-
-			if (currentTriggerPolicies.isEmpty()) {
 
-				// only add the element to its group
-				groupInvokable.processRealElement(nextRecord.getObject());
-				checkForEmptyGroupBuffer(groupInvokable);
+			Object key = keySelector.getKey(nextObject);
 
-				// If central eviction is used, handle it here
-				if (!centralEvictionPolicies.isEmpty()) {
-					evictElements(centralEviction(nextRecord.getObject(), false));
-					deleteOrderForCentralEviction.add(groupInvokable);
-				}
+			StreamDiscretizer<IN> groupDiscretizer = groupedDiscretizers.get(key);
 
-			} else {
-
-				// call user function for all groups
-				for (StreamDiscretizer<IN> group : windowingGroups.values()) {
-					if (group == groupInvokable) {
-						// process real with initialized policies
-						group.processRealElement(nextRecord.getObject(), currentTriggerPolicies);
-					} else {
-						// process like a fake but also initialized with
-						// policies
-						group.externalTriggerFakeElement(nextRecord.getObject(),
-								currentTriggerPolicies);
-					}
-
-					// remove group in case it has an empty buffer
-					// checkForEmptyGroupBuffer(group);
-				}
-
-				// If central eviction is used, handle it here
-				if (!centralEvictionPolicies.isEmpty()) {
-					evictElements(centralEviction(nextRecord.getObject(), true));
-					deleteOrderForCentralEviction.add(groupInvokable);
-				}
+			if (groupDiscretizer == null) {
+				groupDiscretizer = makeNewGroup(key);
+				groupedDiscretizers.put(key, groupDiscretizer);
 			}
 
-			// clear current trigger list
-			currentTriggerPolicies.clear();
+			groupDiscretizer.processRealElement(nextObject);
 
-			// read next record
 			readNext();
 		}
 
-		// Stop all remaining threads from policies
-		for (Thread t : activePolicyThreads) {
-			t.interrupt();
-		}
-
 		// finally trigger the buffer.
-		for (StreamDiscretizer<IN> group : windowingGroups.values()) {
-			group.emitFinalWindow(centralTriggerPolicies);
+		for (StreamDiscretizer<IN> group : groupedDiscretizers.values()) {
+			group.emitFinalWindow();
 		}
 
 	}
@@ -318,178 +86,21 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 	/**
 	 * This method creates a new group. The method gets called in case an
 	 * element arrives which has a key which was not seen before. The method
-	 * created a nested {@link WindowInvokable} and therefore created clones of
-	 * all distributed trigger and eviction policies.
+	 * created a nested {@link StreamDiscretizer} and therefore created clones
+	 * of all distributed trigger and eviction policies.
 	 * 
-	 * @param element
-	 *            The element which leads to the generation of a new group
-	 *            (previously unseen key)
-	 * @throws Exception
-	 *             In case the {@link KeySelector} throws an exception in
-	 *             {@link KeySelector#getKey(Object)}, the exception is not
-	 *             catched by this method.
+	 * @param key
+	 *            The key of the new group.
 	 */
-	private StreamDiscretizer<IN> makeNewGroup(StreamRecord<IN> element) throws Exception {
-		// clone the policies
-		LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-		LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
-		for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
-			clonedDistributedTriggerPolicies.add(trigger.clone());
-		}
-		for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
-			clonedDistributedEvictionPolicies.add(eviction.clone());
-		}
+	private StreamDiscretizer<IN> makeNewGroup(Object key) throws Exception {
 
-		StreamDiscretizer<IN> groupDiscretizer = new StreamDiscretizer<IN>(
-				clonedDistributedTriggerPolicies, clonedDistributedEvictionPolicies);
+		StreamDiscretizer<IN> groupDiscretizer = new StreamDiscretizer<IN>(triggerPolicy.clone(),
+				evictionPolicy.clone());
 
-		groupDiscretizer.setup(taskContext);
+		groupDiscretizer.collector = taskContext.getOutputCollector();
 		groupDiscretizer.open(this.parameters);
-		windowingGroups.put(keySelector.getKey(element.getObject()), groupDiscretizer);
 
 		return groupDiscretizer;
 	}
 
-	@Override
-	public void open(Configuration parameters) throws Exception {
-		super.open(parameters);
-		this.parameters = parameters;
-		for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
-			Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
-			if (target != null) {
-				Thread thread = new Thread(target);
-				activePolicyThreads.add(thread);
-				thread.start();
-			}
-		}
-	};
-
-	/**
-	 * This method is used to notify central eviction policies with a real
-	 * element.
-	 * 
-	 * @param input
-	 *            the real element to notify the eviction policy.
-	 * @param triggered
-	 *            whether a central trigger occurred or not.
-	 * @return The number of elements to be deleted from the buffer.
-	 */
-	private int centralEviction(IN input, boolean triggered) {
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (EvictionPolicy<IN> evictionPolicy : centralEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEviction(input, triggered,
-					deleteOrderForCentralEviction.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
-		return currentMaxEviction;
-	}
-
-	/**
-	 * This method is used to notify active central eviction policies with a
-	 * fake element.
-	 * 
-	 * @param input
-	 *            the fake element to notify the active central eviction
-	 *            policies.
-	 * @return The number of elements to be deleted from the buffer.
-	 */
-	private int centralActiveEviction(Object input) {
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (ActiveEvictionPolicy<IN> evictionPolicy : activeCentralEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input,
-					deleteOrderForCentralEviction.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
-		return currentMaxEviction;
-	}
-
-	/**
-	 * This method is used in central eviction to delete a given number of
-	 * elements from the buffer.
-	 * 
-	 * @param numToEvict
-	 *            number of elements to delete from the virtual central element
-	 *            buffer.
-	 */
-	private void evictElements(int numToEvict) {
-		HashSet<StreamDiscretizer<IN>> usedGroups = new HashSet<StreamDiscretizer<IN>>();
-		for (; numToEvict > 0; numToEvict--) {
-			StreamDiscretizer<IN> currentGroup = deleteOrderForCentralEviction.getFirst();
-			// Do the eviction
-			currentGroup.evictFirst();
-			// Remember groups which possibly have an empty buffer after the
-			// eviction
-			usedGroups.add(currentGroup);
-			try {
-				deleteOrderForCentralEviction.removeFirst();
-			} catch (NoSuchElementException e) {
-				// when buffer is empty, ignore exception and stop deleting
-				break;
-			}
-
-		}
-
-		// Remove groups with empty buffer
-		for (StreamDiscretizer<IN> group : usedGroups) {
-			checkForEmptyGroupBuffer(group);
-		}
-	}
-
-	/**
-	 * Checks if the element buffer of a given windowing group is empty. If so,
-	 * the group will be deleted.
-	 * 
-	 * @param group
-	 *            The windowing group to be checked and and removed in case its
-	 *            buffer is empty.
-	 */
-	private void checkForEmptyGroupBuffer(StreamDiscretizer<IN> group) {
-		if (group.isBufferEmpty()) {
-			windowingGroups.remove(group);
-		}
-	}
-
-	/**
-	 * This callback class allows to handle the the callbacks done by threads
-	 * defined in active trigger policies
-	 * 
-	 * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
-	 */
-	private class WindowingCallback implements ActiveTriggerCallback {
-		private ActiveTriggerPolicy<IN> policy;
-
-		public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
-			this.policy = policy;
-		}
-
-		@Override
-		public void sendFakeElement(Object datapoint) {
-
-			// If central eviction is used, handle it here
-			if (!centralEvictionPolicies.isEmpty()) {
-				evictElements(centralActiveEviction(datapoint));
-			}
-
-			// handle element in groups
-			for (StreamDiscretizer<IN> group : windowingGroups.values()) {
-				group.processFakeElement(datapoint, policy);
-				checkForEmptyGroupBuffer(group);
-			}
-		}
-
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
index a63aee6..d5b4354 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
@@ -18,10 +18,8 @@
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import java.util.LinkedList;
-import java.util.List;
 import java.util.NoSuchElementException;
 
-import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
@@ -36,286 +34,113 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 	 */
 	private static final long serialVersionUID = -8038984294071650730L;
 
-	private LinkedList<TriggerPolicy<IN>> triggerPolicies;
-	private LinkedList<EvictionPolicy<IN>> evictionPolicies;
-	private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
-	private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
-	private LinkedList<Thread> activePolicyTreads;
+	private TriggerPolicy<IN> triggerPolicy;
+	private EvictionPolicy<IN> evictionPolicy;
+	private boolean isActiveTrigger;
+	private boolean isActiveEviction;
+	private Thread activePolicyThread;
 	protected LinkedList<IN> buffer;
-	private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
 
-	/**
-	 * This constructor created a windowing invokable using trigger and eviction
-	 * policies.
-	 * 
-	 * @param userFunction
-	 *            The user defined {@link ReduceFunction}
-	 * @param triggerPolicies
-	 *            A list of {@link TriggerPolicy}s and/or
-	 *            {@link ActiveTriggerPolicy}s
-	 * @param evictionPolicies
-	 *            A list of {@link EvictionPolicy}s and/or
-	 *            {@link ActiveEvictionPolicy}s
-	 */
-	public StreamDiscretizer(LinkedList<TriggerPolicy<IN>> triggerPolicies,
-			LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+	public StreamDiscretizer(TriggerPolicy<IN> triggerPolicy, EvictionPolicy<IN> evictionPolicy) {
 		super(null);
 
-		this.triggerPolicies = triggerPolicies;
-		this.evictionPolicies = evictionPolicies;
+		this.triggerPolicy = triggerPolicy;
+		this.evictionPolicy = evictionPolicy;
 
-		activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
-		for (TriggerPolicy<IN> tp : triggerPolicies) {
-			if (tp instanceof ActiveTriggerPolicy) {
-				activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
-			}
-		}
+		this.isActiveTrigger = triggerPolicy instanceof ActiveTriggerPolicy;
+		this.isActiveEviction = evictionPolicy instanceof ActiveEvictionPolicy;
 
-		activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
-		for (EvictionPolicy<IN> ep : evictionPolicies) {
-			if (ep instanceof ActiveEvictionPolicy) {
-				activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
-			}
-		}
-
-		this.activePolicyTreads = new LinkedList<Thread>();
 		this.buffer = new LinkedList<IN>();
-		this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-	}
-
-	@Override
-	public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
-		super.open(parameters);
-		for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
-			Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
-			if (target != null) {
-				Thread thread = new Thread(target);
-				activePolicyTreads.add(thread);
-				thread.start();
-			}
-		}
-	}
-
-	/**
-	 * This class allows the active trigger threads to call back and push fake
-	 * elements at any time.
-	 */
-	private class WindowingCallback implements ActiveTriggerCallback {
-		private ActiveTriggerPolicy<IN> policy;
-
-		public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
-			this.policy = policy;
-		}
-
-		@Override
-		public void sendFakeElement(Object datapoint) {
-			processFakeElement(datapoint, this.policy);
-		}
-
 	}
 
 	@Override
 	public void invoke() throws Exception {
 
-		// Prevent empty data streams
-		if (readNext() == null) {
-			throw new RuntimeException("DataStream must not be empty");
-		}
-
 		// Continuously run
-		while (nextRecord != null) {
-			processRealElement(nextRecord.getObject());
-
-			// Load next StreamRecord
-			readNext();
+		while (readNext() != null) {
+			processRealElement(nextObject);
 		}
 
-		// Stop all remaining threads from policies
-		for (Thread t : activePolicyTreads) {
-			t.interrupt();
+		if (activePolicyThread != null) {
+			activePolicyThread.interrupt();
 		}
 
-		// finally trigger the buffer.
-		emitFinalWindow(null);
+		emitFinalWindow();
 
 	}
 
 	/**
-	 * This method gets called in case of an grouped windowing in case central
-	 * trigger occurred and the arriving element causing the trigger is not part
-	 * of this group.
-	 * 
-	 * Remark: This is NOT the same as
-	 * {@link StreamDiscretizer#processFakeElement(Object, TriggerPolicy)}! Here
-	 * the eviction using active policies takes place after the call to the UDF.
-	 * Usually it is done before when fake elements get submitted. This special
-	 * behaviour is needed to allow the {@link GroupedStreamDiscretizer} to send
-	 * central triggers to all groups, even if the current element does not
-	 * belong to the group.
+	 * This method processed an arrived real element The method is synchronized
+	 * to ensure that it cannot interleave with
+	 * {@link StreamDiscretizer#triggerOnFakeElement(Object)}
 	 * 
 	 * @param input
-	 *            a fake input element
-	 * @param policies
-	 *            the list of policies which caused the call with this fake
-	 *            element
+	 *            a real input element
 	 */
-	protected synchronized void externalTriggerFakeElement(IN input,
-			List<TriggerPolicy<IN>> policies) {
-
-		// Set the current triggers
-		currentTriggerPolicies.addAll(policies);
-
-		// emit
-		emitWindow();
-
-		// clear the flag collection
-		currentTriggerPolicies.clear();
-
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
+	protected synchronized void processRealElement(IN input) {
 
-		for (int i = 0; i < currentMaxEviction; i++) {
-			try {
-				buffer.removeFirst();
-			} catch (NoSuchElementException e) {
-				// In case no more elements are in the buffer:
-				// Prevent failure and stop deleting.
-				break;
+		if (isActiveTrigger) {
+			ActiveTriggerPolicy<IN> trigger = (ActiveTriggerPolicy<IN>) triggerPolicy;
+			Object[] result = trigger.preNotifyTrigger(input);
+			for (Object in : result) {
+				triggerOnFakeElement(in);
 			}
 		}
-	}
 
-	/**
-	 * This method processed an arrived fake element The method is synchronized
-	 * to ensure that it cannot interleave with
-	 * {@link StreamDiscretizer#processRealElement(Object)}
-	 * 
-	 * @param input
-	 *            a fake input element
-	 * @param currentPolicy
-	 *            the policy which produced this fake element
-	 */
-	protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
-
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
+		boolean isTriggered = false;
 
-		for (int i = 0; i < currentMaxEviction; i++) {
-			try {
-				buffer.removeFirst();
-			} catch (NoSuchElementException e) {
-				// In case no more elements are in the buffer:
-				// Prevent failure and stop deleting.
-				break;
-			}
+		if (triggerPolicy.notifyTrigger(input)) {
+			emitWindow();
+			isTriggered = true;
 		}
 
-		// Set the current trigger
-		currentTriggerPolicies.add(currentPolicy);
+		evict(input, isTriggered);
 
-		// emit
-		emitWindow();
+		buffer.add(input);
 
-		// clear the flag collection
-		currentTriggerPolicies.clear();
 	}
 
 	/**
-	 * This method processed an arrived real element The method is synchronized
-	 * to ensure that it cannot interleave with
-	 * {@link StreamDiscretizer#processFakeElement(Object)}.
+	 * This method triggers on an arrived fake element The method is
+	 * synchronized to ensure that it cannot interleave with
+	 * {@link StreamDiscretizer#processRealElement(Object)}
 	 * 
 	 * @param input
-	 *            a real input element
-	 * @param triggerPolicies
-	 *            Allows to set trigger policies which are maintained
-	 *            externally. This is the case for central policies in
-	 *            {@link GroupedStreamDiscretizer}.
+	 *            a fake input element
 	 */
-	protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
-		this.currentTriggerPolicies.addAll(triggerPolicies);
-		processRealElement(input);
+	protected synchronized void triggerOnFakeElement(Object input) {
+		activeEvict(input);
+		emitWindow();
 	}
 
 	/**
-	 * This method processed an arrived real element The method is synchronized
-	 * to ensure that it cannot interleave with
-	 * {@link StreamDiscretizer#processFakeElement(Object)}
-	 * 
-	 * @param input
-	 *            a real input element
+	 * This method emits the content of the buffer as a new {@link StreamWindow}
 	 */
-	protected synchronized void processRealElement(IN input) {
-
-		// Run the precalls to detect missed windows
-		for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
-			// Remark: In case multiple active triggers are present the ordering
-			// of the different fake elements returned by this triggers becomes
-			// a problem. This might lead to unexpected results...
-			// Should we limit the number of active triggers to 0 or 1?
-			Object[] result = trigger.preNotifyTrigger(input);
-			for (Object in : result) {
-				processFakeElement(in, trigger);
-			}
-		}
+	protected void emitWindow() {
+		StreamWindow<IN> currentWindow = new StreamWindow<IN>();
+		currentWindow.addAll(buffer);
+		collector.collect(currentWindow);
+	}
 
-		// Remember if a trigger occurred
-		boolean isTriggered = false;
+	private void activeEvict(Object input) {
+		int numToEvict = 0;
 
-		// Process the triggers
-		for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
-			if (triggerPolicy.notifyTrigger(input)) {
-				currentTriggerPolicies.add(triggerPolicy);
-			}
+		if (isActiveEviction) {
+			ActiveEvictionPolicy<IN> ep = (ActiveEvictionPolicy<IN>) evictionPolicy;
+			numToEvict = ep.notifyEvictionWithFakeElement(input, buffer.size());
 		}
 
-		// call user function
-		if (!currentTriggerPolicies.isEmpty()) {
-			// emit
-			emitWindow();
-
-			// clear the flag collection
-			currentTriggerPolicies.clear();
+		evictFromBuffer(numToEvict);
+	}
 
-			// remember trigger
-			isTriggered = true;
-		}
+	private void evict(IN input, boolean isTriggered) {
+		int numToEvict = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
 
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-
-		for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
+		evictFromBuffer(numToEvict);
+	}
 
-		for (int i = 0; i < currentMaxEviction; i++) {
+	private void evictFromBuffer(int n) {
+		for (int i = 0; i < n; i++) {
 			try {
 				buffer.removeFirst();
 			} catch (NoSuchElementException e) {
@@ -324,64 +149,48 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 				break;
 			}
 		}
-
-		// Add the current element to the buffer
-		buffer.add(input);
-
-	}
-
-	private void emitWindow() {
-		StreamWindow<IN> currentWindow = new StreamWindow<IN>();
-		currentWindow.addAll(buffer);
-		collector.collect(currentWindow);
 	}
 
 	/**
-	 * This method removes the first element from the element buffer. It is used
-	 * to provide central evictions in {@link GroupedStreamDiscretizer}
+	 * This function emits the partial windows at the end of the stream
 	 */
-	protected synchronized void evictFirst() {
-		try {
-			buffer.removeFirst();
-		} catch (NoSuchElementException e) {
-			// ignore exception
+	protected void emitFinalWindow() {
+		if (!buffer.isEmpty()) {
+			emitWindow();
 		}
 	}
 
-	/**
-	 * This method returns whether the element buffer is empty or not. It is
-	 * used to figure out if a group can be deleted or not when
-	 * {@link GroupedStreamDiscretizer} is used.
-	 * 
-	 * @return true in case the buffer is empty otherwise false.
-	 */
-	protected boolean isBufferEmpty() {
-		return buffer.isEmpty();
-	}
+	@Override
+	public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
+		super.open(parameters);
 
-	/**
-	 * This method does the final reduce at the end of the stream and emits the
-	 * result.
-	 * 
-	 * @param centralTriggerPolicies
-	 *            Allows to set trigger policies which are maintained
-	 *            externally. This is the case for central policies in
-	 *            {@link GroupedStreamDiscretizer}.
-	 */
-	protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
-		if (!buffer.isEmpty()) {
-			currentTriggerPolicies.clear();
+		if (isActiveTrigger) {
+			ActiveTriggerPolicy<IN> tp = (ActiveTriggerPolicy<IN>) triggerPolicy;
 
-			if (centralTriggerPolicies != null) {
-				currentTriggerPolicies.addAll(centralTriggerPolicies);
+			Runnable runnable = tp.createActiveTriggerRunnable(new WindowingCallback());
+			if (activePolicyThread != null) {
+				activePolicyThread = new Thread(runnable);
+				activePolicyThread.start();
 			}
+		}
+	}
 
-			for (TriggerPolicy<IN> policy : triggerPolicies) {
-				currentTriggerPolicies.add(policy);
-			}
+	@Override
+	public String toString() {
+		return buffer.toString();
+	}
 
-			emitWindow();
+	/**
+	 * This class allows the active trigger thread to call back and push fake
+	 * elements at any time.
+	 */
+	private class WindowingCallback implements ActiveTriggerCallback {
+
+		@Override
+		public void sendFakeElement(Object datapoint) {
+			triggerOnFakeElement(datapoint);
 		}
+
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
index b37bf47..51f2718 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindow.java
@@ -143,6 +143,11 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 	}
 
 	@Override
+	public boolean equals(Object o) {
+		return super.equals(o);
+	}
+
+	@Override
 	public void collect(T record) {
 		add(record);
 	}
@@ -153,6 +158,14 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 
 	@Override
 	public String toString() {
-		return super.toString() + " " + windowID + " (" + numberOfParts + ")";
+		return super.toString();
+	}
+
+	public static <R> StreamWindow<R> fromElements(R... elements) {
+		StreamWindow<R> window = new StreamWindow<R>();
+		for (R element : elements) {
+			window.add(element);
+		}
+		return window;
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
index 9f223c3..5f5e7d2 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
@@ -43,6 +43,7 @@ public class WindowFlattener<T> extends ChainableInvokable<StreamWindow<T>, T> {
 
 	@Override
 	public void collect(StreamWindow<T> record) {
+		nextObject = record;
 		callUserFunctionAndLogException();
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
deleted file mode 100755
index 757f6f6..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/PrintTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api;
-
-import java.io.Serializable;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.util.TestStreamEnvironment;
-import org.junit.Test;
-
-public class PrintTest implements Serializable {
-
-	private static final long serialVersionUID = 1L;
-	private static final long MEMORYSIZE = 32;
-
-	private static final class IdentityMap implements MapFunction<Long, Long> {
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public Long map(Long value) throws Exception {
-			return value;
-		}
-	}
-
-	private static final class FilterAll implements FilterFunction<Long> {
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public boolean filter(Long value) throws Exception {
-			return true;
-		}
-	}
-
-	@Test
-	public void test() throws Exception {
-		StreamExecutionEnvironment env = new TestStreamEnvironment(1, MEMORYSIZE);
-		env.generateSequence(1, 10).map(new IdentityMap()).filter(new FilterAll()).print();
-		env.execute();
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
new file mode 100644
index 0000000..8a6a7ce
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class GroupedStreamDiscretizerTest {
+
+	KeySelector<Tuple2<Integer, String>, ?> keySelector = new KeySelector<Tuple2<Integer, String>, String>() {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public String getKey(Tuple2<Integer, String> value) throws Exception {
+			return value.f1;
+		}
+	};
+
+	/**
+	 * Test for not active distributed triggers with single field
+	 */
+	@Test
+	public void groupedDiscretizerTest() {
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(2);
+		inputs.add(2);
+		inputs.add(3);
+		inputs.add(4);
+		inputs.add(5);
+		inputs.add(10);
+		inputs.add(11);
+		inputs.add(11);
+
+		Set<StreamWindow<Integer>> expected = new HashSet<StreamWindow<Integer>>();
+		expected.add(StreamWindow.fromElements(2, 2));
+		expected.add(StreamWindow.fromElements(1, 3));
+		expected.add(StreamWindow.fromElements(5, 11));
+		expected.add(StreamWindow.fromElements(4, 10));
+		expected.add(StreamWindow.fromElements(11));
+
+		KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer getKey(Integer value) {
+				return value % 2;
+			}
+		};
+
+		CloneableTriggerPolicy<Integer> trigger = new CountTriggerPolicy<Integer>(2);
+		CloneableEvictionPolicy<Integer> eviction = new TumblingEvictionPolicy<Integer>();
+
+		GroupedStreamDiscretizer<Integer> discretizer = new GroupedStreamDiscretizer<Integer>(
+				keySelector, trigger, eviction);
+
+		List<StreamWindow<Integer>> result = MockContext.createAndExecute(discretizer, inputs);
+		assertEquals(expected, new HashSet<StreamWindow<Integer>>(result));
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/412779fa/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java
deleted file mode 100644
index 1bbdcf6..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class GroupedWindowInvokableTest {
-
-	KeySelector<Tuple2<Integer, String>, ?> keySelector = new KeySelector<Tuple2<Integer, String>, String>() {
-
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public String getKey(Tuple2<Integer, String> value) throws Exception {
-			return value.f1;
-		}
-	};
-
-	/**
-	 * Tests that illegal arguments result in failure. The following cases are
-	 * tested: 1) having no trigger 2) having no eviction 3) having neither
-	 * eviction nor trigger 4) having both, central and distributed eviction.
-	 */
-	@Test
-	public void testGroupedWindowInvokableFailTest() {
-
-		// create dummy reduce function
-		ReduceFunction<Object> userFunction = new ReduceFunction<Object>() {
-
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Object reduce(Object value1, Object value2) throws Exception {
-				return null;
-			}
-		};
-
-		// create dummy keySelector
-		KeySelector<Object, Object> keySelector = new KeySelector<Object, Object>() {
-
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Object getKey(Object value) throws Exception {
-				return null;
-			}
-		};
-
-		// create policy lists
-		LinkedList<CloneableEvictionPolicy<Object>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<Object>>();
-		LinkedList<CloneableTriggerPolicy<Object>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<Object>>();
-		LinkedList<EvictionPolicy<Object>> centralEvictionPolicies = new LinkedList<EvictionPolicy<Object>>();
-		LinkedList<TriggerPolicy<Object>> centralTriggerPolicies = new LinkedList<TriggerPolicy<Object>>();
-
-		// empty trigger and policy lists should fail
-		try {
-			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (1)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// null for trigger and policy lists should fail
-		try {
-			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector, null, null, null,
-					null);
-			fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (2)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// empty eviction should still fail
-		centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
-		distributedTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
-		try {
-			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance without any eviction policy should cause an UnsupportedOperationException but didn't. (3)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// empty trigger should still fail
-		centralTriggerPolicies.clear();
-		distributedTriggerPolicies.clear();
-		centralEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
-		try {
-			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance without any trigger policy should cause an UnsupportedOperationException but didn't. (4)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// having both, central and distributed eviction, at the same time
-		// should fail
-		centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
-		distributedEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
-		try {
-			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance with central and distributed eviction should cause an UnsupportedOperationException but didn't. (4)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-	}
-
-	/**
-	 * Test for not active distributed triggers with single field
-	 */
-	@Test
-	public void testGroupedWindowInvokableDistributedTriggerSimple() {
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(1);
-		inputs.add(5);
-		inputs.add(5);
-		inputs.add(5);
-		inputs.add(1);
-		inputs.add(1);
-		inputs.add(5);
-		inputs.add(1);
-		inputs.add(5);
-
-		List<Integer> expectedDistributedEviction = new ArrayList<Integer>();
-		expectedDistributedEviction.add(15);
-		expectedDistributedEviction.add(3);
-		expectedDistributedEviction.add(3);
-		expectedDistributedEviction.add(15);
-
-		List<Integer> expectedCentralEviction = new ArrayList<Integer>();
-		expectedCentralEviction.add(2);
-		expectedCentralEviction.add(5);
-		expectedCentralEviction.add(15);
-		expectedCentralEviction.add(2);
-		expectedCentralEviction.add(5);
-		expectedCentralEviction.add(2);
-		expectedCentralEviction.add(5);
-		expectedCentralEviction.add(1);
-		expectedCentralEviction.add(5);
-
-		LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-		// Trigger on every 2nd element, but the first time after the 3rd
-		triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
-		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
-		// On every 2nd element, remove the oldest 2 elements, but the first
-		// time after the 3rd element
-		evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
-		LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
-
-		ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer getKey(Integer value) {
-				return value;
-			}
-		};
-
-		GroupedStreamDiscretizer<Integer, Integer> invokable = new GroupedStreamDiscretizer<Integer, Integer>(
-				reduceFunction, keySelector, triggers, evictions, centralTriggers, null);
-
-		List<Integer> result = MockContext.createAndExecute(invokable, inputs);
-
-		List<Integer> actual = new LinkedList<Integer>();
-		for (Integer current : result) {
-			actual.add(current);
-		}
-
-		assertEquals(new HashSet<Integer>(expectedDistributedEviction),
-				new HashSet<Integer>(actual));
-		assertEquals(expectedDistributedEviction.size(), actual.size());
-
-		// Run test with central eviction
-		triggers.clear();
-		centralTriggers.add(new CountTriggerPolicy<Integer>(2, -1));
-		LinkedList<EvictionPolicy<Integer>> centralEvictions = new LinkedList<EvictionPolicy<Integer>>();
-		centralEvictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
-		invokable = new GroupedStreamDiscretizer<Integer, Integer>(reduceFunction, keySelector,
-				triggers, null, centralTriggers, centralEvictions);
-
-		result = MockContext.createAndExecute(invokable, inputs);
-		actual = new LinkedList<Integer>();
-		for (Integer current : result) {
-			actual.add(current);
-		}
-
-		assertEquals(new HashSet<Integer>(expectedCentralEviction), new HashSet<Integer>(actual));
-		assertEquals(expectedCentralEviction.size(), actual.size());
-	}
-
-	/**
-	 * Test for non active distributed triggers with separated key field
-	 */
-	@Test
-	public void testGroupedWindowInvokableDistributedTriggerComplex() {
-		List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
-		inputs2.add(new Tuple2<Integer, String>(1, "a"));
-		inputs2.add(new Tuple2<Integer, String>(0, "b"));
-		inputs2.add(new Tuple2<Integer, String>(2, "a"));
-		inputs2.add(new Tuple2<Integer, String>(-1, "a"));
-		inputs2.add(new Tuple2<Integer, String>(-2, "a"));
-		inputs2.add(new Tuple2<Integer, String>(10, "a"));
-		inputs2.add(new Tuple2<Integer, String>(2, "b"));
-		inputs2.add(new Tuple2<Integer, String>(1, "a"));
-
-		List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
-		expected2.add(new Tuple2<Integer, String>(-1, "a"));
-		expected2.add(new Tuple2<Integer, String>(-2, "a"));
-		expected2.add(new Tuple2<Integer, String>(0, "b"));
-
-		LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
-		// Trigger on every 2nd element, but the first time after the 3rd
-		triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
-
-		LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
-		// On every 2nd element, remove the oldest 2 elements, but the first
-		// time after the 3rd element
-		evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
-
-		LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
-
-		GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>>(
-				new ReduceFunction<Tuple2<Integer, String>>() {
-					private static final long serialVersionUID = 1L;
-
-					@Override
-					public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
-							Tuple2<Integer, String> value2) throws Exception {
-						if (value1.f0 <= value2.f0) {
-							return value1;
-						} else {
-							return value2;
-						}
-					}
-				}, keySelector, triggers, evictions, centralTriggers, null);
-
-		List<Tuple2<Integer, String>> result = MockContext.createAndExecute(invokable2, inputs2);
-
-		List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
-		for (Tuple2<Integer, String> current : result) {
-			actual2.add(current);
-		}
-
-		assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
-				new HashSet<Tuple2<Integer, String>>(actual2));
-		assertEquals(expected2.size(), actual2.size());
-	}
-
-	/**
-	 * Test for active centralized trigger
-	 */
-	@Test
-	public void testGroupedWindowInvokableCentralActiveTrigger() {
-
-		List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
-		inputs.add(new Tuple2<Integer, String>(1, "a"));
-		inputs.add(new Tuple2<Integer, String>(1, "b"));
-		inputs.add(new Tuple2<Integer, String>(1, "c"));
-		inputs.add(new Tuple2<Integer, String>(2, "a"));
-		inputs.add(new Tuple2<Integer, String>(2, "b"));
-		inputs.add(new Tuple2<Integer, String>(2, "c"));
-		inputs.add(new Tuple2<Integer, String>(2, "b"));
-		inputs.add(new Tuple2<Integer, String>(2, "a"));
-		inputs.add(new Tuple2<Integer, String>(2, "c"));
-		inputs.add(new Tuple2<Integer, String>(3, "c"));
-		inputs.add(new Tuple2<Integer, String>(3, "a"));
-		inputs.add(new Tuple2<Integer, String>(3, "b"));
-		inputs.add(new Tuple2<Integer, String>(4, "a"));
-		inputs.add(new Tuple2<Integer, String>(4, "b"));
-		inputs.add(new Tuple2<Integer, String>(4, "c"));
-		inputs.add(new Tuple2<Integer, String>(5, "c"));
-		inputs.add(new Tuple2<Integer, String>(5, "a"));
-		inputs.add(new Tuple2<Integer, String>(5, "b"));
-		inputs.add(new Tuple2<Integer, String>(10, "b"));
-		inputs.add(new Tuple2<Integer, String>(10, "a"));
-		inputs.add(new Tuple2<Integer, String>(10, "c"));
-		inputs.add(new Tuple2<Integer, String>(11, "a"));
-		inputs.add(new Tuple2<Integer, String>(11, "a"));
-		inputs.add(new Tuple2<Integer, String>(11, "c"));
-		inputs.add(new Tuple2<Integer, String>(11, "c"));
-		inputs.add(new Tuple2<Integer, String>(11, "b"));
-		inputs.add(new Tuple2<Integer, String>(11, "b"));
-
-		// Expected result:
-		// For each group (a,b and c):
-		// 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
-		// 12-12-5-10-32
-
-		List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
-		expected.add(new Tuple2<Integer, String>(12, "a"));
-		expected.add(new Tuple2<Integer, String>(12, "b"));
-		expected.add(new Tuple2<Integer, String>(12, "c"));
-		expected.add(new Tuple2<Integer, String>(12, "a"));
-		expected.add(new Tuple2<Integer, String>(12, "b"));
-		expected.add(new Tuple2<Integer, String>(12, "c"));
-		expected.add(new Tuple2<Integer, String>(5, "a"));
-		expected.add(new Tuple2<Integer, String>(5, "b"));
-		expected.add(new Tuple2<Integer, String>(5, "c"));
-		expected.add(new Tuple2<Integer, String>(10, "a"));
-		expected.add(new Tuple2<Integer, String>(10, "b"));
-		expected.add(new Tuple2<Integer, String>(10, "c"));
-		expected.add(new Tuple2<Integer, String>(32, "a"));
-		expected.add(new Tuple2<Integer, String>(32, "b"));
-		expected.add(new Tuple2<Integer, String>(32, "c"));
-
-		Timestamp<Tuple2<Integer, String>> myTimeStamp = new Timestamp<Tuple2<Integer, String>>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public long getTimestamp(Tuple2<Integer, String> value) {
-				return value.f0;
-			}
-		};
-
-		TimestampWrapper<Tuple2<Integer, String>> myTimeStampWrapper = new TimestampWrapper<Tuple2<Integer, String>>(
-				myTimeStamp, 1);
-
-		ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
-					Tuple2<Integer, String> value2) throws Exception {
-				return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
-			}
-		};
-
-		LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
-		// Trigger every 2 time units but delay the first trigger by 2 (First
-		// trigger after 4, then every 2)
-		triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
-
-		LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
-		// Always delete all elements older then 4
-		evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
-
-		LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
-
-		GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>>(
-				myReduceFunction, keySelector, distributedTriggers, evictions, triggers, null);
-
-		ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
-		for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
-				new HashSet<Tuple2<Integer, String>>(result));
-		assertEquals(expected.size(), result.size());
-
-		// repeat the test with central eviction. The result should be the same.
-		triggers.clear();
-		triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
-		evictions.clear();
-		LinkedList<EvictionPolicy<Tuple2<Integer, String>>> centralEvictions = new LinkedList<EvictionPolicy<Tuple2<Integer, String>>>();
-		centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
-
-		invokable = new GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>>(
-				myReduceFunction, keySelector, distributedTriggers, evictions, triggers,
-				centralEvictions);
-
-		result = new ArrayList<Tuple2<Integer, String>>();
-		for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
-				new HashSet<Tuple2<Integer, String>>(result));
-		assertEquals(expected.size(), result.size());
-	}
-
-	/**
-	 * Test for multiple centralized trigger
-	 */
-	@Test
-	public void testGroupedWindowInvokableMultipleCentralTrigger() {
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		triggers.add(new CountTriggerPolicy<Integer>(8));
-		triggers.add(new CountTriggerPolicy<Integer>(5));
-
-		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
-		// The active wrapper causes eviction even on (fake) elements which
-		// triggered, but does not belong to the group.
-		evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
-				new TumblingEvictionPolicy<Integer>()));
-
-		LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(2);
-		inputs.add(2);
-		inputs.add(2);
-		inputs.add(1);
-		// 1st Trigger: 2;6
-		inputs.add(2);
-		inputs.add(1);
-		inputs.add(2);
-		// 2nd Trigger: 1;4
-		inputs.add(2);
-		inputs.add(1);
-		// Final: 1,2
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(2);
-		expected.add(6);
-		expected.add(4);
-		expected.add(1);
-		expected.add(2);
-		expected.add(1);
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		GroupedStreamDiscretizer<Integer, Integer> invokable = new GroupedStreamDiscretizer<Integer, Integer>(
-				myReduceFunction, new KeySelector<Integer, Integer>() {
-					private static final long serialVersionUID = 1L;
-
-					@Override
-					public Integer getKey(Integer value) {
-						return value;
-					}
-				}, distributedTriggers, evictions, triggers, null);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
-		assertEquals(expected.size(), result.size());
-	}
-
-	/**
-	 * Test for combination of centralized trigger and distributed trigger at
-	 * the same time
-	 */
-	@Test
-	public void testGroupedWindowInvokableCentralAndDistrTrigger() {
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		triggers.add(new CountTriggerPolicy<Integer>(8));
-		triggers.add(new CountTriggerPolicy<Integer>(5));
-
-		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
-		// The active wrapper causes eviction even on (fake) elements which
-		// triggered, but does not belong to the group.
-		evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
-				new TumblingEvictionPolicy<Integer>()));
-
-		LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-		distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(2);
-		inputs.add(2);
-		// local on 2 => 4
-		inputs.add(2);
-		inputs.add(1);
-		// local on 1 => 2
-		// and 1st Central: 2;2
-		// SUMS up to 2;2
-		inputs.add(2);
-		// local on 2 => 2
-		inputs.add(1);
-		inputs.add(2);
-		// 2nd Central: 1;2
-		inputs.add(2);
-		inputs.add(1);
-		// Final: 1,2
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(4);
-		expected.add(2);
-		expected.add(2);
-		expected.add(2);
-		expected.add(1);
-		expected.add(2);
-		expected.add(1);
-		expected.add(2);
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		GroupedStreamDiscretizer<Integer, Integer> invokable = new GroupedStreamDiscretizer<Integer, Integer>(
-				myReduceFunction, new KeySelector<Integer, Integer>() {
-					private static final long serialVersionUID = 1L;
-
-					@Override
-					public Integer getKey(Integer value) {
-						return value;
-					}
-				}, distributedTriggers, evictions, triggers, null);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
-		assertEquals(expected.size(), result.size());
-	}
-
-}


[11/20] flink git commit: [streaming] Documentations updated to match the reworked windowing semantics

Posted by mb...@apache.org.
[streaming] Documentations updated to match the reworked windowing semantics


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/f571ece6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/f571ece6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/f571ece6

Branch: refs/heads/master
Commit: f571ece6c2a1728bdd90195309546771686e0277
Parents: 4470207
Author: Gyula Fora <gy...@apache.org>
Authored: Sat Feb 14 11:41:09 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 docs/streaming_guide.md                         | 100 +++++++++++++------
 .../streaming/api/datastream/DataStream.java    |  44 ++++----
 .../api/datastream/DiscretizedStream.java       |  29 ++----
 .../api/datastream/WindowedDataStream.java      |  74 ++++++++------
 .../api/function/RichWindowMapFunction.java     |   9 ++
 .../api/function/WindowMapFunction.java         |   7 ++
 .../windowing/GroupedStreamDiscretizer.java     |  10 +-
 .../windowing/GroupedTimeDiscretizer.java       |   4 +
 .../operator/windowing/StreamDiscretizer.java   |   6 ++
 .../operator/windowing/WindowFlattener.java     |   4 +
 .../operator/windowing/WindowMapper.java        |   5 +
 .../operator/windowing/WindowMerger.java        |   5 +
 .../operator/windowing/WindowPartitioner.java   |   7 +-
 .../operator/windowing/WindowReducer.java       |   5 +
 .../streaming/api/windowing/StreamWindow.java   |  81 +++++++++++++++
 .../windowbuffer/BasicWindowBuffer.java         |   4 +
 .../windowbuffer/CompletePreAggregator.java     |   4 +
 .../windowbuffer/TumblingGroupedPreReducer.java |   3 +
 .../windowbuffer/TumblingPreReducer.java        |   3 +
 .../windowing/windowbuffer/WindowBuffer.java    |   4 +
 .../flink/streaming/api/scala/DataStream.scala  |  18 ++--
 .../api/scala/WindowedDataStream.scala          |  29 +++---
 22 files changed, 322 insertions(+), 133 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/docs/streaming_guide.md
----------------------------------------------------------------------
diff --git a/docs/streaming_guide.md b/docs/streaming_guide.md
index 3476390..e409b49 100644
--- a/docs/streaming_guide.md
+++ b/docs/streaming_guide.md
@@ -42,7 +42,7 @@ Add the following dependency to your `pom.xml` to use the Flink Streaming.
 <dependency>
     <groupId>org.apache.flink</groupId>
     <artifactId>flink-streaming-core</artifactId>
-    <version>{{site.FLINK_VERSION_SHORT}}</version>
+    <version>{{site.FLINK_VERSION_STABLE}}</version>
 </dependency>
 ~~~
 
@@ -162,11 +162,11 @@ Usage: `dataStream.shuffle()`
  * *Distribute*: Distribute partitioning directs the output data stream to the next operator in a round-robin fashion, achieving a balanced distribution.
 Usage: `dataStream.distribute()`
  * *Field/Key*: Field/Key partitioning partitions the output data stream based on the hash code of a selected key of the tuples. Data points with the same key are directed to the same operator instance. The user can define keys by field positions (for tuple and array types), field expressions (for Pojo types) and custom keys using the `KeySelector` interface. 
-Usage: `dataStream.groupBy(keys)`
+Usage: `dataStream.partitionBy(keys)`
  * *Broadcast*: Broadcast partitioning sends the output data stream to all parallel instances of the next operator.
 Usage: `dataStream.broadcast()`
- * *Global*: All data are sent to the first instance of the next processing operator. Use this option with care to avoid serious performance bottlenecks.
-Usage: `dataStream.global()`
+ * *Global*: All data points end up at the same operator instance. To achieve this use the parallelism setting of the corresponding operator.
+Usage: `operator.setParallelism(1)`
 
 ### Sources
 
@@ -292,16 +292,24 @@ There is also an option to apply user defined aggregations with the usage of the
 
 ### Window operators
 
-Flink streaming provides very flexible windowing semantics to create arbitrary windows (also referred to as discretizations or slices) of the data streams and apply reduction or aggregation operations on the windows acquired. Windowing can be used for instance to create rolling aggregations of the most recent N elements, where N could be defined by Time, Count or any arbitrary user defined measure.
+Flink streaming provides very flexible windowing semantics to create arbitrary windows (also referred to as discretizations or slices) of the data streams and apply reduce, map or aggregation operations on the windows acquired. Windowing can be used for instance to create rolling aggregations of the most recent N elements, where N could be defined by Time, Count or any arbitrary user defined measure.
 
-The user can control the size (eviction) of the windows and the frequency of reduction or aggregation calls (triggers) on them in an intuitive API:
+The user can control the size (eviction) of the windows and the frequency of reduction or aggregation calls (trigger) on them in an intuitive API (some examples):
 
+ * `dataStream.window(eviction).every(trigger).reduceWindow(…)`
+ * `dataStream.window(…).every(…).mapWindow(…).flatten()`
+ * `dataStream.window(…).every(…).groupBy(…).aggregate(…).getDiscretizedStream()`
 
- * `dataStream.window(…).every(…).reduce(…)`
- * `dataStream.window(…).every(…).reduceGroup(…)`
- * `dataStream.window(…).every(…).aggregate(…)`
+The core abstraction of the Windowing semantics is the `WindowedDataStream` and the `StreamWindow`. The WindowedDataStream which is created when we call the `.window(…)` method of the DataStream, represents the windowed discretisation of the underlying stream. The user can think about it simply as a `DataStream<StreamWindow<T>>` where additional API functions are supplied to provide efficient transformations of individual windows. 
 
-The next example would create windows that hold elements of the last 5 seconds, and the user defined aggregation/reduce is executed on the windows every second (sliding the window by 1 second):
+The result of a window transformation is again a `WindowedDataStream` which can also be used to further transform the resulting windows. In this sense, window transformations define mapping from `StreamWindow -> StreamWindow’`.
+
+The user have different ways of working further with a result of a window operation:
+ * `windowedDataStream.flatten()` - Which streams out the results element wise and returns a `DataStream<T>` where T is the type of the underlying windowed stream
+ * `windowedDataStream.getDiscretizedStream()` - Which returns a `DataStream<StreamWindow<T>>` for applying some advanced logic on the stream windows itself
+ * Calling any window transformation further transforms the windows
+
+The next example would create windows that hold elements of the last 5 seconds, and the user defined transformation would be executed on the windows every second (sliding the window by 1 second):
 
 ~~~java
 dataStream.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(1, TimeUnit.SECONDS))
@@ -313,7 +321,7 @@ This approach is often referred to as policy based windowing. Different policies
 dataStream.window(Count.of(100)).every(Time.of(1, TimeUnit.MINUTES))
 ~~~
 
-The user can also omit the `.every(…)` call which results in a tumbling window emptying the window after every aggregation call.
+The user can also omit the `.every(…)` call which results in a tumbling window emptying the window after every transformation call.
 
 Several predefined policies are provided in the API, including delta-based, count-based and time-based policies. These can be accessed through the static methods provided by the `PolicyHelper` classes:
 
@@ -328,26 +336,22 @@ The policy based windowing is a highly flexible way to specify stream discretisa
 
  * `TriggerPolicy` defines when to trigger the reduce UDF on the current window and emit the result. In the API it completes a window statement such as: `.window(…).every(…)`, while the triggering policy is passed within `every`. 
 
-When multiple triggers are used, the reduction or aggregation is executed at every trigger.
-
 Several predefined policies are provided in the API, including delta-based, punctuation based, count-based and time-based policies. Policies are in general UDFs and can implement any custom behaviour.
 
  * `EvictionPolicy` defines the length of a window as a means of a predicate for evicting tuples when they are no longer needed. In the API this can be defined by the `.window(…)` operation on a stream. There are mostly the same predefined policy types provided as for trigger policies.
 
-When multiple evictions are used the strictest one controls the elements in the window. For instance in the call `dataStream.window(Count.of(5), Time.of(1,TimeUnit.SECONDS)).every(…)` produces a window of maximum 5 elements which have arrived in the last second.
-
-In addition to the `dataStream.window(…).every(…)` style users can specifically pass the list of trigger and eviction policies during the window call:
+In addition to the `dataStream.window(…).every(…)` style users can specifically pass the trigger and eviction policies during the window call:
 
 ~~~java
-dataStream.window(ListOfTriggerPolicies, ListOfEvictionPolicies)
+dataStream.window(TriggerPolicy, EvictionPolicy)
 ~~~
 
-By default most triggers can only trigger when a new element arrives. This might not be suitable for all the use-cases, especially when time based windowing is applied. To also provide trigering between elements so called active policies can be used. The predefined time-based policies are already implemented in such a way and can hold as an example for user defined active policy implementations. 
+By default most triggers can only trigger when a new element arrives. This might not be suitable for all the use-cases, especially when time based windowing is applied. To also provide triggering between elements so called active policies can be used. The predefined time-based policies are already implemented in such a way and can hold as an example for user defined active policy implementations. 
 
 Time-based trigger and eviction policies can work with user defined `TimeStamp` implementations, these policies already cover most use cases.
  
 #### Reduce on windowed data streams
-The transformation calls a user-defined `ReduceFunction` at every trigger on the records currently in the window. The user can also use the different streaming aggregations.
+The `WindowedDataStream<T>.reduceWindow(ReduceFunction<T>)` transformation calls the user-defined `ReduceFunction` at every trigger on the records currently in the window. The user can also use the different pre-implemented streaming aggregations: `sum, min, max, minBy, maxBy`
 
 A window reduce that sums the elements in the last minute with 10 seconds slide interval:
 
@@ -355,19 +359,21 @@ A window reduce that sums the elements in the last minute with 10 seconds slide
 dataStream.window(Time.of(1, TimeUnit.MINUTES)).every(Time.of(10,TimeUnit.SECONDS)).sum(field);
 ~~~
 
-#### ReduceGroup on windowed data streams
-The transformation calls a `GroupReduceFunction` for each data batch or data window similarly as a reduce, but providing access to all elements in the window.
+#### Map on windowed data streams
+The `WindowedDataStream<T>.mapWindow(WindowMapFunction<T,O>)` transformation calls  `WindowMapFunction.mapWindow(…)` for each `StreamWindow` in the discretised stream providing access to all elements in the window through the iterable interface. At each function call the output `StreamWindow<O>` will consist of all the elements collected to the collector. This allows a straightforward way of mapping one stream window to another.
 
 ~~~java
-dataStream.window(…).every(…).reduceGroup(reducer);
+windowedDataStream.mapWindow(windowMapFunction)
 ~~~
 
 #### Grouped operations on windowed data streams
-Calling the `.groupBy(fields)` method on a windowed stream groups the elements by the given fields inside the windows. The window sizes (evictions) and slide sizes (triggers) will be calculated on the whole stream (in a central fashion), but the user defined functions will be applied on a per group basis.
+Calling the `.groupBy(…)` method on a windowed stream groups the elements by the given fields inside the stream windows. The window sizes (evictions) and slide sizes (triggers) will be calculated on the whole stream (in a global fashion), but the user defined functions will be applied on a per group basis. This means that for a call `windowedStream.groupBy(…).reduceWindow(…)` will transform each window into another window consisting of as many elements as groups, with the reduced values per key. Similarly the `mapWindow` transformation is applied per group as well.
 
-The user can also create windows and triggers on a per group basis calling `.window(…).every(…)` on an already grouped data stream. To highlight the differences let us look at to examples.
+The user can also create discretisation on a per group basis calling `.window(…).every(…)` on an already grouped data stream. This will apply the discretisation logic independently for each key.
 
-To get the maximal value by key on the last 100 elements we use the first approach:
+To highlight the differences let us look at two examples.
+
+To get the maximal value for each key on the last 100 elements (global) we use the first approach:
 
 ~~~java
 dataStream.window(Count.of(100)).every(…).groupBy(groupingField).max(field);
@@ -383,9 +389,40 @@ dataStream.groupBy(groupingField).window(Count.of(100)).every(…).max(field);
 
 This will create separate windows for different keys and apply the trigger and eviction policies on a per group basis.
 
+#### Applying multiple transformations on a window
+Using the `WindowedDataStream` abstraction we can apply several transformations one after another on the discretised streams without having to re-discretise it:
+
+~~~java
+dataStream.window(Count.of(1000)).groupBy(firstKey).mapWindow(…).groupBy(secondKey).reduceWindow(…).flatten()
+~~~
+
+The above call would create global windows of 1000 elements group it by the first key and then apply a mapWindow transformation. The resulting windowed stream will then be grouped by the second key and further reduced. The results of the reduce transformation are then flattened.
+
+Notice here that we only defined the window size once at the beginning of the transformation. This means that anything that happens afterwards (`.groupBy(firstKey).mapWindow(…).groupBy(secondKey).reduceWindow(…)`) happens inside the 1000 element windows. Of course the mapWindow might reduce the number of elements but the idea is that each transformation still corresponds to the same 1000 elements in the original stream.
+
+#### Global vs local discretisation
+By default all window discretisation calls (`dataStream.window(…)`) define global windows meaning that a global window of count 100 will contain the last 100 elements arrived at the discretisation operator in order. In most cases (except for Time) this means that the operator doing the actual discretisation needs to have a degree of parallelism of 1 to be able to correctly execute the discretisation logic. 
+
+Sometimes it is enough to create local discretisations, which allows the discretiser to run in parallel and apply the given discretisation logic at every discretiser instance. To allow local discretisation use the `.local()` method of the windowed data stream.
+
+Example:
+
+~~~java
+dataStream.window(Count.of(100)).maxBy(field);
+~~~
+
+This would create global windows of 100 elements (Count discretises with parallelism of 1) and return the record with the max value by the selected field, while
+
+~~~java
+dataStream.window(Count.of(100)).local().maxBy(field);
+~~~
+
+this would create several count discretisers (as defined by the environment parallelism) and compute the max values accordingly.
+
+
 ### Temporal database style operators
 
-While database style operators like joins (on key) and crosses are hard to define properly on data streams, a straight forward implementation is to apply these operators on windows of the data streams.
+While database style operators like joins (on key) and crosses are hard to define properly on data streams, a straight forward implementation is to apply these operators on windows of the data streams. 
 
 Currently join and cross operators are supported on time windows.
 
@@ -406,6 +443,7 @@ The Cross transformation combines two DataStreams into one DataStreams. It build
 dataStream1.cross(dataStream2).onWindow(windowing_params);
 ~~~
 
+Please note that this is currently not integrated with the windowing semantics, integration is work in progress.
 
 ### Co operators
 
@@ -475,7 +513,7 @@ DataStream<Integer> odd = split.select("odd");
 
 In the above example the data stream named ‘even’ will only contain elements that are directed to the output named “even”. The user can of course further transform these new stream by for example squaring only the even elements.
 
-Data streams only receive the elements directed to selected output names. The user can also select multiple output names by `splitStream.select(“output1”, “output2”…)`. It is common that a stream listens to all the outputs, by simply applying the transformation on the split data stream without select provides this functionality.
+Data streams only receive the elements directed to selected output names. The user can also select multiple output names by `splitStream.select(“output1”, “output2”…)`. It is common that a stream listens to all the outputs, so `split.selectAll()` provides this functionality without having to select all names.
 
 The outputs of an operator are directed by implementing a selector function (implementing the `OutputSelector` interface):
 
@@ -857,7 +895,7 @@ After installing Docker an image can be pulled for each connector. Containers ca
 For the easiest set up create a jar with all the dependencies of the *flink-streaming-connectors* project.
 
 ~~~batch
-cd /PATH/TO/GIT/flink/flink-staging/flink-streaming-connectors
+cd /PATH/TO/GIT/incubator-flink/flink-addons/flink-streaming-connectors
 mvn assembly:assembly
 ~~~batch
 
@@ -974,14 +1012,14 @@ Now a terminal started running from the image with all the necessary configurati
 
 To have the latest version of Flink type:
 ~~~batch
-cd /git/flink/
+cd /git/incubator-flink/
 git pull
 ~~~
 
 Then build the code with:
 
 ~~~batch
-cd /git/flink/flink-staging/flink-streaming/flink-streaming-connectors/
+cd /git/incubator-flink/flink-addons/flink-streaming/flink-streaming-connectors/
 mvn install -DskipTests
 ~~~
 
@@ -1009,4 +1047,4 @@ In the example there are to connectors. One that sends messages to Flume and one
 <DATE> INFO flume.FlumeTopology: String: <five> arrived from Flume
 ~~~
 
-[Back to top](#top)
+[Back to top](#top)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
index 71a97f8..f221e4b 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DataStream.java
@@ -850,14 +850,15 @@ public class DataStream<OUT> {
 
 	/**
 	 * Create a {@link WindowedDataStream} that can be used to apply
-	 * transformation like {@link WindowedDataStream#reduce} or aggregations on
-	 * preset chunks(windows) of the data stream. To define the windows one or
-	 * more {@link WindowingHelper} such as {@link Time}, {@link Count} and
+	 * transformation like {@link WindowedDataStream#reduceWindow},
+	 * {@link WindowedDataStream#mapWindow} or aggregations on preset
+	 * chunks(windows) of the data stream. To define windows a
+	 * {@link WindowingHelper} such as {@link Time}, {@link Count} and
 	 * {@link Delta} can be used.</br></br> When applied to a grouped data
 	 * stream, the windows (evictions) and slide sizes (triggers) will be
 	 * computed on a per group basis. </br></br> For more advanced control over
 	 * the trigger and eviction policies please refer to
-	 * {@link #window(triggers, evicters)} </br> </br> For example to create a
+	 * {@link #window(trigger, eviction)} </br> </br> For example to create a
 	 * sum every 5 seconds in a tumbling fashion:</br>
 	 * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).sum(field)} </br></br> To
 	 * create sliding windows use the
@@ -867,33 +868,34 @@ public class DataStream<OUT> {
 	 * {@code ds.window(Time.of(5, TimeUnit.SECONDS)).every(Time.of(3,
 	 *       TimeUnit.SECONDS)).sum(field)}
 	 * 
-	 * @param policyHelpers
+	 * @param policyHelper
 	 *            Any {@link WindowingHelper} such as {@link Time},
-	 *            {@link Count} and {@link Delta} to define the window.
+	 *            {@link Count} and {@link Delta} to define the window size.
 	 * @return A {@link WindowedDataStream} providing further operations.
 	 */
 	@SuppressWarnings({ "rawtypes", "unchecked" })
-	public WindowedDataStream<OUT> window(WindowingHelper policyHelpers) {
-		return new WindowedDataStream<OUT>(this, policyHelpers);
+	public WindowedDataStream<OUT> window(WindowingHelper policyHelper) {
+		return new WindowedDataStream<OUT>(this, policyHelper);
 	}
 
 	/**
 	 * Create a {@link WindowedDataStream} using the given {@link TriggerPolicy}
-	 * s and {@link EvictionPolicy}s. Windowing can be used to apply
-	 * transformation like {@link WindowedDataStream#reduce} or aggregations on
-	 * preset chunks(windows) of the data stream.</br></br>For most common
-	 * use-cases please refer to {@link #window(WindowingHelper...)}
-	 * 
-	 * @param triggers
-	 *            The list of {@link TriggerPolicy}s that will determine how
-	 *            often the user function is called on the window.
-	 * @param evicters
-	 *            The list of {@link EvictionPolicy}s that will determine the
-	 *            number of elements in each time window.
+	 * and {@link EvictionPolicy}. Windowing can be used to apply transformation
+	 * like {@link WindowedDataStream#reduceWindow},
+	 * {@link WindowedDataStream#mapWindow} or aggregations on preset
+	 * chunks(windows) of the data stream.</br></br>For most common use-cases
+	 * please refer to {@link #window(WindowingHelper)}
+	 * 
+	 * @param trigger
+	 *            The {@link TriggerPolicy} that will determine how often the
+	 *            user function is called on the window.
+	 * @param eviction
+	 *            The {@link EvictionPolicy} that will determine the number of
+	 *            elements in each time window.
 	 * @return A {@link WindowedDataStream} providing further operations.
 	 */
-	public WindowedDataStream<OUT> window(TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> evicter) {
-		return new WindowedDataStream<OUT>(this, trigger, evicter);
+	public WindowedDataStream<OUT> window(TriggerPolicy<OUT> trigger, EvictionPolicy<OUT> eviction) {
+		return new WindowedDataStream<OUT>(this, trigger, eviction);
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
index 7ab7c2c..bc9f5b9 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.api.datastream;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichReduceFunction;
 import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
@@ -67,16 +66,6 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		return discretizedStream;
 	}
 
-	/**
-	 * Applies a reduce transformation on the windowed data stream by reducing
-	 * the current window at every trigger.The user can also extend the
-	 * {@link RichReduceFunction} to gain access to other features provided by
-	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
-	 * 
-	 * @param reduceFunction
-	 *            The reduce function that will be applied to the windows.
-	 * @return The transformed DataStream
-	 */
 	@Override
 	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
 
@@ -92,18 +81,6 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		}
 	}
 
-	/**
-	 * Applies a reduceGroup transformation on the windowed data stream by
-	 * reducing the current window at every trigger. In contrast with the
-	 * standard binary reducer, with reduceGroup the user can access all
-	 * elements of the window at the same time through the iterable interface.
-	 * The user can also extend the to gain access to other features provided by
-	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
-	 * 
-	 * @param windowMapFunction
-	 *            The reduce function that will be applied to the windows.
-	 * @return The transformed DataStream
-	 */
 	@Override
 	public <R> DiscretizedStream<R> mapWindow(WindowMapFunction<OUT, R> windowMapFunction) {
 
@@ -238,4 +215,10 @@ public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
 		return new DiscretizedStream<OUT>(discretizedStream.copy(), groupByKey, transformation);
 	}
 
+	@Override
+	public WindowedDataStream<OUT> local() {
+		throw new UnsupportedOperationException(
+				"Local discretisation can only be applied after defining the discretisation logic");
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 4da12ac..b63cb96 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.operators.Keys;
+import org.apache.flink.streaming.api.function.RichWindowMapFunction;
 import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
@@ -53,10 +54,11 @@ import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 import org.apache.flink.streaming.util.keys.KeySelectorUtil;
 
 /**
- * A {@link WindowedDataStream} represents a data stream that has been divided
- * into windows (predefined chunks). User defined function such as
+ * A {@link WindowedDataStream} represents a data stream that has been
+ * discretised into windows. User defined function such as
  * {@link #reduceWindow(ReduceFunction)}, {@link #mapWindow()} or aggregations
- * can be applied to the windows.
+ * can be applied to the windows. The results of these transformations are also
+ * WindowedDataStreams of the same discretisation unit.
  * 
  * @param <OUT>
  *            The output type of the {@link WindowedDataStream}
@@ -64,9 +66,7 @@ import org.apache.flink.streaming.util.keys.KeySelectorUtil;
 public class WindowedDataStream<OUT> {
 
 	protected enum WindowTransformation {
-
 		REDUCEWINDOW, MAPWINDOW, NONE;
-
 		private Function UDF;
 
 		public WindowTransformation with(Function UDF) {
@@ -132,8 +132,8 @@ public class WindowedDataStream<OUT> {
 	 * </br></br> The user function in this case will be called on the 5 most
 	 * recent elements every 2 seconds
 	 * 
-	 * @param policyHelpers
-	 *            The policies that define the triggering frequency
+	 * @param policyHelper
+	 *            The policy that define the triggering frequency
 	 * 
 	 * @return The windowed data stream with triggering set
 	 */
@@ -151,7 +151,7 @@ public class WindowedDataStream<OUT> {
 	/**
 	 * Groups the elements of the {@link WindowedDataStream} by the given key
 	 * positions. The window sizes (evictions) and slide sizes (triggers) will
-	 * be calculated on the whole stream (in a central fashion), but the user
+	 * be calculated on the whole stream (in a global fashion), but the user
 	 * defined functions will be applied on a per group basis. </br></br> To get
 	 * windows and triggers on a per group basis apply the
 	 * {@link DataStream#window} operator on an already grouped data stream.
@@ -171,7 +171,7 @@ public class WindowedDataStream<OUT> {
 	/**
 	 * Groups the elements of the {@link WindowedDataStream} by the given field
 	 * expressions. The window sizes (evictions) and slide sizes (triggers) will
-	 * be calculated on the whole stream (in a central fashion), but the user
+	 * be calculated on the whole stream (in a global fashion), but the user
 	 * defined functions will be applied on a per group basis. </br></br> To get
 	 * windows and triggers on a per group basis apply the
 	 * {@link DataStream#window} operator on an already grouped data stream.
@@ -191,7 +191,7 @@ public class WindowedDataStream<OUT> {
 	/**
 	 * Groups the elements of the {@link WindowedDataStream} using the given
 	 * {@link KeySelector}. The window sizes (evictions) and slide sizes
-	 * (triggers) will be calculated on the whole stream (in a central fashion),
+	 * (triggers) will be calculated on the whole stream (in a global fashion),
 	 * but the user defined functions will be applied on a per group basis.
 	 * </br></br> To get windows and triggers on a per group basis apply the
 	 * {@link DataStream#window} operator on an already grouped data stream.
@@ -212,11 +212,10 @@ public class WindowedDataStream<OUT> {
 	}
 
 	/**
-	 * Sets the windowed computations local, so that the windowing and reduce or
-	 * aggregation logic will be computed for each parallel instance of this
-	 * operator
+	 * Sets the window discretisation local, meaning that windows will be
+	 * created in parallel at environment parallelism.
 	 * 
-	 * @return The local windowed data stream
+	 * @return The WindowedDataStream with local discretisation
 	 */
 	public WindowedDataStream<OUT> local() {
 		WindowedDataStream<OUT> out = copy();
@@ -224,11 +223,24 @@ public class WindowedDataStream<OUT> {
 		return out;
 	}
 
+	/**
+	 * Returns the {@link DataStream} of {@link StreamWindow}s which represent
+	 * the discretised stream. There is no ordering guarantee for the received
+	 * windows.
+	 * 
+	 * @return The discretised stream
+	 */
 	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
 		return discretize(WindowTransformation.NONE, new BasicWindowBuffer<OUT>())
 				.getDiscretizedStream();
 	}
 
+	/**
+	 * Flattens the results of the window computations and streams out the
+	 * window elements.
+	 * 
+	 * @return The data stream consisting of the individual records.
+	 */
 	public DataStream<OUT> flatten() {
 		return dataStream;
 	}
@@ -259,15 +271,16 @@ public class WindowedDataStream<OUT> {
 	}
 
 	/**
-	 * Applies a reduceGroup transformation on the windowed data stream by
-	 * reducing the current window at every trigger. In contrast with the
-	 * standard binary reducer, with reduceGroup the user can access all
+	 * Applies a mapWindow transformation on the windowed data stream by calling
+	 * the mapWindow function on the window at every trigger. In contrast with
+	 * the standard binary reducer, with mapWindow allows the user to access all
 	 * elements of the window at the same time through the iterable interface.
-	 * The user can also extend the to gain access to other features provided by
-	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
+	 * The user can also extend the {@link RichWindowMapFunction} to gain access
+	 * to other features provided by the
+	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
 	 * 
 	 * @param windowMapFunction
-	 *            The reduce function that will be applied to the windows.
+	 *            The function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
 	public <R> WindowedDataStream<R> mapWindow(WindowMapFunction<OUT, R> windowMapFunction) {
@@ -276,18 +289,21 @@ public class WindowedDataStream<OUT> {
 	}
 
 	/**
-	 * Applies a reduceGroup transformation on the windowed data stream by
-	 * reducing the current window at every trigger. In contrast with the
-	 * standard binary reducer, with reduceGroup the user can access all
+	 * Applies a mapWindow transformation on the windowed data stream by calling
+	 * the mapWindow function on the window at every trigger. In contrast with
+	 * the standard binary reducer, with mapWindow allows the user to access all
 	 * elements of the window at the same time through the iterable interface.
-	 * The user can also extend the to gain access to other features provided by
-	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
-	 * </br> </br> This version of reduceGroup uses user supplied
-	 * typeinformation for serializaton. Use this only when the system is unable
-	 * to detect type information using: {@link #mapWindow()}
+	 * The user can also extend the {@link RichWindowMapFunction} to gain access
+	 * to other features provided by the
+	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+	 * </br> </br> This version of mapWindow uses user supplied typeinformation
+	 * for serializaton. Use this only when the system is unable to detect type
+	 * information.
 	 * 
 	 * @param windowMapFunction
-	 *            The reduce function that will be applied to the windows.
+	 *            The function that will be applied to the windows.
+	 * @param outType
+	 *            The output type of the operator.
 	 * @return The transformed DataStream
 	 */
 	public <R> WindowedDataStream<R> mapWindow(WindowMapFunction<OUT, R> windowMapFunction,

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java
index ac2a19e..e34a2f9 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/RichWindowMapFunction.java
@@ -18,8 +18,17 @@
 package org.apache.flink.streaming.api.function;
 
 import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.RichFunction;
+import org.apache.flink.streaming.api.datastream.WindowedDataStream;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.util.Collector;
 
+/**
+ * Abstract class for defining rich mapWindow transformation to be applied on
+ * {@link WindowedDataStream}s. The mapWindow function will be called on each
+ * {@link StreamWindow}.</p> In addition the user can access the functionality
+ * provided by the {@link RichFunction} interface.
+ */
 public abstract class RichWindowMapFunction<IN, OUT> extends AbstractRichFunction implements
 		WindowMapFunction<IN, OUT> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java
index 273d731..eb5f6c3 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/function/WindowMapFunction.java
@@ -20,8 +20,15 @@ package org.apache.flink.streaming.api.function;
 import java.io.Serializable;
 
 import org.apache.flink.api.common.functions.Function;
+import org.apache.flink.streaming.api.datastream.WindowedDataStream;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.util.Collector;
 
+/**
+ * Interface for defining mapWindow transformation to be applied on
+ * {@link WindowedDataStream}s. The mapWindow function will be called on each
+ * {@link StreamWindow}.
+ */
 public interface WindowMapFunction<T, O> extends Function, Serializable {
 
 	void mapWindow(Iterable<T> values, Collector<O> out) throws Exception;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
index 58d7adb..d6bbb08 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
@@ -28,11 +28,15 @@ import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 
+/**
+ * This invokable represents the grouped discretization step of a window
+ * transformation. The user supplied eviction and trigger policies are applied
+ * on a per group basis to create the {@link StreamWindow} that will be further
+ * transformed in the next stages. </p> To allow pre-aggregations supply an
+ * appropriate {@link WindowBuffer}
+ */
 public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>> {
 
-	/**
-	 * Auto-generated serial version UID
-	 */
 	private static final long serialVersionUID = -3469545957144404137L;
 
 	protected KeySelector<IN, ?> keySelector;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
index 5363c10..a9f78b6 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
@@ -22,6 +22,10 @@ import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 
+/**
+ * A specialized {@link GroupedStreamDiscretizer} to be used with time only
+ * policies
+ */
 public class GroupedTimeDiscretizer<IN> extends GroupedStreamDiscretizer<IN> {
 
 	private static final long serialVersionUID = -3469545957144404137L;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
index a14058a..d053dd9 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
@@ -26,6 +26,12 @@ import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
 import org.apache.flink.streaming.api.windowing.windowbuffer.WindowBuffer;
 
+/**
+ * This invokable represents the discretization step of a window transformation.
+ * The user supplied eviction and trigger policies are applied to create the
+ * {@link StreamWindow} that will be further transformed in the next stages.
+ * </p> To allow pre-aggregations supply an appropriate {@link WindowBuffer}
+ */
 public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>> {
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
index 7eaea58..edefeef 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
@@ -20,6 +20,10 @@ package org.apache.flink.streaming.api.invokable.operator.windowing;
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 
+/**
+ * This invokable flattens the results of the window transformations by
+ * outputing the elements of the {@link StreamWindow} one-by-one
+ */
 public class WindowFlattener<T> extends ChainableInvokable<StreamWindow<T>, T> {
 
 	public WindowFlattener() {

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
index de93fab..3dfd59d 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
@@ -18,10 +18,15 @@
 package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.streaming.api.datastream.WindowedDataStream;
 import org.apache.flink.streaming.api.function.WindowMapFunction;
 import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 
+/**
+ * This invokable is used to apply mapWindow transformations on
+ * {@link WindowedDataStream}s.
+ */
 public class WindowMapper<IN, OUT> extends MapInvokable<StreamWindow<IN>, StreamWindow<OUT>> {
 
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
index 8601d06..a58bb9f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
@@ -23,6 +23,11 @@ import java.util.Map;
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 
+/**
+ * This invokable merges together the different partitions of the
+ * {@link StreamWindow}s used to merge the results of parallel transformations
+ * that belong in the same window.
+ */
 public class WindowMerger<T> extends ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
 
 	private Map<Integer, StreamWindow<T>> windows;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
index ea4451e..e010af4 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
@@ -21,8 +21,11 @@ import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 
-public class WindowPartitioner<T> extends
-		ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
+/**
+ * This invokable applies either split or key partitioning depending on the
+ * transformation.
+ */
+public class WindowPartitioner<T> extends ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
 
 	private KeySelector<T, ?> keySelector;
 	private int numberOfSplits;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
index 3a4bb69..d7182de 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
@@ -19,9 +19,14 @@ package org.apache.flink.streaming.api.invokable.operator.windowing;
 
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.datastream.WindowedDataStream;
 import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 
+/**
+ * This invokable is used to apply reduceWindow transformations on
+ * {@link WindowedDataStream}s.
+ */
 public class WindowReducer<IN> extends MapInvokable<StreamWindow<IN>, StreamWindow<IN>> {
 
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
index 988058c..74ed62e 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindow.java
@@ -25,8 +25,17 @@ import java.util.Random;
 
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.WindowedDataStream;
 import org.apache.flink.util.Collector;
 
+/**
+ * Core abstraction for representing windows for {@link WindowedDataStream}s.
+ * The user can apply transformations on these windows with the appropriate
+ * {@link WindowedDataStream} methods. </p> Each stream window consists of a
+ * random ID, a number representing the number of partitions for this specific
+ * window (ID) and the elements itself. The ID and number of parts will be used
+ * to merge the subwindows after distributed transformations.
+ */
 public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 
 	private static final long serialVersionUID = -5150196421193988403L;
@@ -36,25 +45,54 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 
 	public int numberOfParts;
 
+	/**
+	 * Creates a new window with a random id
+	 */
 	public StreamWindow() {
 		this(rnd.nextInt(), 1);
 	}
 
+	/**
+	 * Creates a new window with the specific id
+	 * 
+	 * @param windowID
+	 *            ID of the window
+	 */
 	public StreamWindow(int windowID) {
 		this(windowID, 1);
 	}
 
+	/**
+	 * Creates a new window with the given id and number of parts
+	 * 
+	 * @param windowID
+	 * @param numberOfParts
+	 */
 	public StreamWindow(int windowID, int numberOfParts) {
 		super();
 		this.windowID = windowID;
 		this.numberOfParts = numberOfParts;
 	}
 
+	/**
+	 * Creates a shallow copy of the window
+	 * 
+	 * @param window
+	 *            The window to be copied
+	 */
 	public StreamWindow(StreamWindow<T> window) {
 		this(window.windowID, window.numberOfParts);
 		addAll(window);
 	}
 
+	/**
+	 * Creates a deep copy of the window using the given serializer
+	 * 
+	 * @param window
+	 *            The window to be copied
+	 * @param serializer
+	 *            The serializer used for copying the records.
+	 */
 	public StreamWindow(StreamWindow<T> window, TypeSerializer<T> serializer) {
 		this(window.windowID, window.numberOfParts);
 		for (T element : window) {
@@ -62,6 +100,14 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 		}
 	}
 
+	/**
+	 * Partitions the window using the given keyselector. A subwindow will be
+	 * created for each key.
+	 * 
+	 * @param keySelector
+	 *            The keyselector used for extracting keys.
+	 * @return A list of the subwindows
+	 */
 	public List<StreamWindow<T>> partitionBy(KeySelector<T, ?> keySelector) throws Exception {
 		Map<Object, StreamWindow<T>> partitions = new HashMap<Object, StreamWindow<T>>();
 
@@ -85,6 +131,13 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 		return output;
 	}
 
+	/**
+	 * Splits the window into n equal (if possible) sizes.
+	 * 
+	 * @param n
+	 *            Number of desired partitions
+	 * @return The list of subwindows.
+	 */
 	public List<StreamWindow<T>> split(int n) {
 		int numElements = size();
 		if (n > numElements) {
@@ -116,10 +169,24 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 		return this;
 	}
 
+	/**
+	 * Checks whether this window can be merged with the given one.
+	 * 
+	 * @param otherWindow
+	 *            The window to test
+	 * @return Window compatibility
+	 */
 	public boolean compatibleWith(StreamWindow<T> otherWindow) {
 		return this.windowID == otherWindow.windowID && this.numberOfParts > 1;
 	}
 
+	/**
+	 * Merges compatible windows together.
+	 * 
+	 * @param windows
+	 *            Windows to merge
+	 * @return Merged window
+	 */
 	public static <R> StreamWindow<R> merge(StreamWindow<R>... windows) {
 		StreamWindow<R> window = new StreamWindow<R>(windows[0]);
 		for (int i = 1; i < windows.length; i++) {
@@ -134,6 +201,13 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 		return window;
 	}
 
+	/**
+	 * Merges compatible windows together.
+	 * 
+	 * @param windows
+	 *            Windows to merge
+	 * @return Merged window
+	 */
 	public static <R> StreamWindow<R> merge(List<StreamWindow<R>> windows) {
 		if (windows.isEmpty()) {
 			throw new RuntimeException("Need at least one window to merge");
@@ -171,6 +245,13 @@ public class StreamWindow<T> extends ArrayList<T> implements Collector<T> {
 		return super.toString();
 	}
 
+	/**
+	 * Creates a new {@link StreamWindow} with random id from the given elements
+	 * 
+	 * @param elements
+	 *            The elements contained in the resulting window
+	 * @return The window
+	 */
 	public static <R> StreamWindow<R> fromElements(R... elements) {
 		StreamWindow<R> window = new StreamWindow<R>();
 		for (R element : elements) {

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java
index 9d5676c..458de41 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java
@@ -23,6 +23,10 @@ import java.util.NoSuchElementException;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.util.Collector;
 
+/**
+ * Basic window buffer that stores the elements in a simple list without any
+ * pre-aggregation.
+ */
 public class BasicWindowBuffer<T> implements WindowBuffer<T> {
 
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java
index a01f97c..59bd974 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java
@@ -18,6 +18,10 @@
 
 package org.apache.flink.streaming.api.windowing.windowbuffer;
 
+/**
+ * Interface for marking window pre-aggregators that fully process the window so
+ * that no further reduce step is necessary afterwards.
+ */
 public interface CompletePreAggregator {
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java
index eef8bed..7403ffe 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java
@@ -26,6 +26,9 @@ import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.util.Collector;
 
+/**
+ * Grouped pre-reducer for tumbling eviction polciy.
+ */
 public class TumblingGroupedPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
 
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java
index b755e73..35cf60e 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java
@@ -22,6 +22,9 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.util.Collector;
 
+/**
+ * Non-grouped pre-reducer for tumbling eviction policy.
+ */
 public class TumblingPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
 
 	private static final long serialVersionUID = 1L;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
index 1a45194..e0429ab 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
@@ -22,6 +22,10 @@ import java.io.Serializable;
 import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.util.Collector;
 
+/**
+ * Interface for defining specialized buffers to store/emit window data.
+ * Pre-aggregators should be implemented using this interface.
+ */
 public interface WindowBuffer<T> extends Serializable, Cloneable {
 
 	public void store(T element) throws Exception;

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
index f6e92aa..3ab4ff1 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/DataStream.scala
@@ -442,9 +442,9 @@ class DataStream[T](javaStream: JavaStream[T]) {
 
   /**
    * Create a WindowedDataStream that can be used to apply
-   * transformation like .reduce(...) or aggregations on
-   * preset chunks(windows) of the data stream. To define the windows one or
-   * more WindowingHelper-s such as Time, Count and
+   * transformation like .reduceWindow(...) or aggregations on
+   * preset chunks(windows) of the data stream. To define the windows a
+   * WindowingHelper such as Time, Count and
    * Delta can be used.</br></br> When applied to a grouped data
    * stream, the windows (evictions) and slide sizes (triggers) will be
    * computed on a per group basis. </br></br> For more advanced control over
@@ -455,14 +455,14 @@ class DataStream[T](javaStream: JavaStream[T]) {
     javaStream.window(windowingHelper)
 
   /**
-   * Create a WindowedDataStream using the given TriggerPolicy-s and EvictionPolicy-s.
-   * Windowing can be used to apply transformation like .reduce(...) or aggregations on
-   * preset chunks(windows) of the data stream.</br></br>For most common
-   * use-cases please refer to window(WindowingHelper[_]*)
+   * Create a WindowedDataStream using the given Trigger and Eviction policies.
+   * Windowing can be used to apply transformation like .reduceWindow(...) or 
+   * aggregations on preset chunks(windows) of the data stream.</br></br>For most common
+   * use-cases please refer to window(WindowingHelper[_])
    *
    */
-  def window(trigger: TriggerPolicy[T], evicter: EvictionPolicy[T]):
-    WindowedDataStream[T] = javaStream.window(trigger, evicter)
+  def window(trigger: TriggerPolicy[T], eviction: EvictionPolicy[T]):
+    WindowedDataStream[T] = javaStream.window(trigger, eviction)
 
   /**
    *

http://git-wip-us.apache.org/repos/asf/flink/blob/f571ece6/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
index 33d71be..b072197 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -49,7 +49,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
   /**
    * Groups the elements of the WindowedDataStream using the given
    * field positions. The window sizes (evictions) and slide sizes
-   * (triggers) will be calculated on the whole stream (in a central fashion),
+   * (triggers) will be calculated on the whole stream (in a global fashion),
    * but the user defined functions will be applied on a per group basis.
    * </br></br> To get windows and triggers on a per group basis apply the
    * DataStream.window(...) operator on an already grouped data stream.
@@ -60,7 +60,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
   /**
    * Groups the elements of the WindowedDataStream using the given
    * field expressions. The window sizes (evictions) and slide sizes
-   * (triggers) will be calculated on the whole stream (in a central fashion),
+   * (triggers) will be calculated on the whole stream (in a global fashion),
    * but the user defined functions will be applied on a per group basis.
    * </br></br> To get windows and triggers on a per group basis apply the
    * DataStream.window(...) operator on an already grouped data stream.
@@ -72,7 +72,7 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
   /**
    * Groups the elements of the WindowedDataStream using the given
    * KeySelector function. The window sizes (evictions) and slide sizes
-   * (triggers) will be calculated on the whole stream (in a central fashion),
+   * (triggers) will be calculated on the whole stream (in a global fashion),
    * but the user defined functions will be applied on a per group basis.
    * </br></br> To get windows and triggers on a per group basis apply the
    * DataStream.window(...) operator on an already grouped data stream.
@@ -88,16 +88,15 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
   }
   
   /**
-   * Sets the computations local meaning that the windowing and reduce or
-   * aggregation logic will be computed for each parallel instance of this
-   * operator
+   * Sets the window discretisation local, meaning that windows will be
+   * created in parallel at environment parallelism.
    * 
    */
   def local(): WindowedDataStream[T] = javaStream.local
  
   /**
    * Flattens the result of a window transformation returning the stream of window
-   * contents elementwise
+   * contents elementwise.
    */
   def flatten(): DataStream[T] = javaStream.flatten()
   
@@ -135,12 +134,12 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
   }
 
   /**
-   * Applies a reduceGroup transformation on the windowed data stream by reducing
-   * the current window at every trigger. In contrast with the simple binary reduce operator,
-   * groupReduce exposes the whole window through the Iterable interface.
+   * Applies a mapWindow transformation on the windowed data stream by calling the mapWindow
+   * method on current window at every trigger. In contrast with the simple binary reduce 
+   * operator, mapWindow exposes the whole window through the Iterable interface.
    * </br>
    * </br>
-   * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+   * Whenever possible try to use reduceWindow instead of mapWindow for increased efficiency
    */
   def mapWindow[R: ClassTag: TypeInformation](reducer: WindowMapFunction[T, R]):
   WindowedDataStream[R] = {
@@ -151,12 +150,12 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
   }
 
   /**
-   * Applies a reduceGroup transformation on the windowed data stream by reducing
-   * the current window at every trigger. In contrast with the simple binary reduce operator,
-   * groupReduce exposes the whole window through the Iterable interface.
+   * Applies a mapWindow transformation on the windowed data stream by calling the mapWindow
+   * method on current window at every trigger. In contrast with the simple binary reduce 
+   * operator, mapWindow exposes the whole window through the Iterable interface.
    * </br>
    * </br>
-   * Whenever possible try to use reduce instead of groupReduce for increased efficiency
+   * Whenever possible try to use reduceWindow instead of mapWindow for increased efficiency
    */
   def mapWindow[R: ClassTag: TypeInformation](fun: (Iterable[T], Collector[R]) => Unit):
   WindowedDataStream[R] = {


[05/20] flink git commit: [FLINK-1176] [streaming] Added invokables for modular windowing tranformations

Posted by mb...@apache.org.
[FLINK-1176] [streaming] Added invokables for modular windowing tranformations


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1146f64c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1146f64c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1146f64c

Branch: refs/heads/master
Commit: 1146f64c01c89f6790aef7375eb60e389e0071f4
Parents: 667f819
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Feb 4 17:26:16 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:07 2015 +0100

----------------------------------------------------------------------
 .../windowing/GroupedStreamDiscretizer.java     | 495 +++++++++++++++++++
 .../operator/windowing/StreamDiscretizer.java   | 387 +++++++++++++++
 .../operator/windowing/WindowFlattener.java     |  49 ++
 .../operator/windowing/WindowMapper.java        |  57 +++
 .../operator/windowing/WindowMerger.java        |  69 +++
 .../operator/windowing/WindowPartitioner.java   |  74 +++
 .../operator/windowing/WindowReducer.java       |  62 +++
 7 files changed, 1193 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/1146f64c/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
new file mode 100644
index 0000000..6178f48
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.streamrecord.StreamRecord;
+import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+/**
+ * This invokable allows windowing based on {@link TriggerPolicy} and
+ * {@link EvictionPolicy} instances including their active and cloneable
+ * versions. It is additionally aware of the creation of windows per group.
+ * 
+ * A {@link KeySelector} is used to specify the key position or key extraction.
+ * The {@link ReduceFunction} will be executed on each group separately.
+ * Policies might either be centralized or distributed. It is not possible to
+ * use central and distributed eviction policies at the same time. A distributed
+ * policy have to be a {@link CloneableTriggerPolicy} or
+ * {@link CloneableEvictionPolicy} as it will be cloned to have separated
+ * instances for each group. At the startup time the distributed policies will
+ * be stored as sample, and only clones of them will be used to maintain the
+ * groups. Therefore, each group starts with the initial policy states.
+ * 
+ * While a distributed policy only gets notified with the elements belonging to
+ * the respective group, a centralized policy get notified with all arriving
+ * elements. When a centralized trigger occurred, all groups get triggered. This
+ * is done by submitting the element which caused the trigger as real element to
+ * the groups it belongs to and as fake element to all other groups. Within the
+ * groups the element might be further processed, causing more triggers,
+ * prenotifications of active distributed policies and evictions like usual.
+ * 
+ * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+ * implement the
+ * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+ * method. Fake elements created on prenotification will be forwarded to all
+ * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
+ * it forwards/distributed calls all groups.
+ * 
+ * @param <IN>
+ *            The type of input elements handled by this operator invokable.
+ */
+public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>> {
+
+	/**
+	 * Auto-generated serial version UID
+	 */
+	private static final long serialVersionUID = -3469545957144404137L;
+
+	private KeySelector<IN, ?> keySelector;
+	private Configuration parameters;
+	private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies;
+	private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies;
+	private LinkedList<ActiveEvictionPolicy<IN>> activeCentralEvictionPolicies;
+	private LinkedList<EvictionPolicy<IN>> centralEvictionPolicies;
+	private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies;
+	private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies;
+	private Map<Object, StreamDiscretizer<IN>> windowingGroups;
+	private LinkedList<Thread> activePolicyThreads;
+	private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
+	private LinkedList<StreamDiscretizer<IN>> deleteOrderForCentralEviction;
+
+	/**
+	 * This constructor creates an instance of the grouped windowing invokable.
+	 * 
+	 * A {@link KeySelector} is used to specify the key position or key
+	 * extraction. The {@link ReduceFunction} will be executed on each group
+	 * separately. Policies might either be centralized or distributed. It is
+	 * not possible to use central and distributed eviction policies at the same
+	 * time. A distributed policy have to be a {@link CloneableTriggerPolicy} or
+	 * {@link CloneableEvictionPolicy} as it will be cloned to have separated
+	 * instances for each group. At the startup time the distributed policies
+	 * will be stored as sample, and only clones of them will be used to
+	 * maintain the groups. Therefore, each group starts with the initial policy
+	 * states.
+	 * 
+	 * While a distributed policy only gets notified with the elements belonging
+	 * to the respective group, a centralized policy get notified with all
+	 * arriving elements. When a centralized trigger occurred, all groups get
+	 * triggered. This is done by submitting the element which caused the
+	 * trigger as real element to the groups it belongs to and as fake element
+	 * to all other groups. Within the groups the element might be further
+	 * processed, causing more triggers, prenotifications of active distributed
+	 * policies and evictions like usual.
+	 * 
+	 * Central policies can be instance of {@link ActiveTriggerPolicy} and also
+	 * implement the
+	 * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
+	 * method. Fake elements created on prenotification will be forwarded to all
+	 * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
+	 * that it forwards/distributed calls all groups.
+	 * 
+	 * @param userFunction
+	 *            The user defined function.
+	 * @param keySelector
+	 *            A key selector to extract the key for the groups from the
+	 *            input data.
+	 * @param distributedTriggerPolicies
+	 *            Trigger policies to be distributed and maintained individually
+	 *            within each group.
+	 * @param distributedEvictionPolicies
+	 *            Eviction policies to be distributed and maintained
+	 *            individually within each group. Note that there cannot be
+	 *            both, central and distributed eviction policies at the same
+	 *            time.
+	 * @param centralTriggerPolicies
+	 *            Trigger policies which will only exist once at a central
+	 *            place. In case a central policy triggers, it will cause all
+	 *            groups to be emitted. (Remark: Empty groups cannot be emitted.
+	 *            If only one element is contained a group, this element itself
+	 *            is returned as aggregated result.)
+	 * @param centralEvictionPolicies
+	 *            Eviction which will only exist once at a central place. Note
+	 *            that there cannot be both, central and distributed eviction
+	 *            policies at the same time. The central eviction policy will
+	 *            work on an simulated element buffer containing all elements no
+	 *            matter which group they belong to.
+	 */
+	public GroupedStreamDiscretizer(KeySelector<IN, ?> keySelector,
+			LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
+			LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
+			LinkedList<TriggerPolicy<IN>> centralTriggerPolicies,
+			LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
+
+		super(null);
+
+		this.keySelector = keySelector;
+
+		// handle the triggers
+		if (centralTriggerPolicies != null) {
+			this.centralTriggerPolicies = centralTriggerPolicies;
+			this.activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+
+			for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
+				if (trigger instanceof ActiveTriggerPolicy) {
+					this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
+				}
+			}
+		} else {
+			this.centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+		}
+
+		if (distributedTriggerPolicies != null) {
+			this.distributedTriggerPolicies = distributedTriggerPolicies;
+		} else {
+			this.distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
+		}
+
+		if (distributedEvictionPolicies != null) {
+			this.distributedEvictionPolicies = distributedEvictionPolicies;
+		} else {
+			this.distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
+		}
+
+		this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+
+		if (centralEvictionPolicies != null) {
+			this.centralEvictionPolicies = centralEvictionPolicies;
+
+			for (EvictionPolicy<IN> eviction : centralEvictionPolicies) {
+				if (eviction instanceof ActiveEvictionPolicy) {
+					this.activeCentralEvictionPolicies.add((ActiveEvictionPolicy<IN>) eviction);
+				}
+			}
+		} else {
+			this.centralEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
+		}
+
+		this.windowingGroups = new HashMap<Object, StreamDiscretizer<IN>>();
+		this.activePolicyThreads = new LinkedList<Thread>();
+		this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+		this.deleteOrderForCentralEviction = new LinkedList<StreamDiscretizer<IN>>();
+
+		// check that not both, central and distributed eviction, is used at the
+		// same time.
+		if (!this.centralEvictionPolicies.isEmpty() && !this.distributedEvictionPolicies.isEmpty()) {
+			throw new UnsupportedOperationException(
+					"You can only use either central or distributed eviction policies but not both at the same time.");
+		}
+
+		// Check that there is at least one trigger and one eviction policy
+		if (this.centralEvictionPolicies.isEmpty() && this.distributedEvictionPolicies.isEmpty()) {
+			throw new UnsupportedOperationException(
+					"You have to define at least one eviction policy");
+		}
+		if (this.centralTriggerPolicies.isEmpty() && this.distributedTriggerPolicies.isEmpty()) {
+			throw new UnsupportedOperationException(
+					"You have to define at least one trigger policy");
+		}
+
+	}
+
+	@Override
+	public void invoke() throws Exception {
+		// Prevent empty data streams
+		if (readNext() == null) {
+			throw new RuntimeException("DataStream must not be empty");
+		}
+
+		// Continuously run
+		while (nextRecord != null) {
+			StreamDiscretizer<IN> groupInvokable = windowingGroups.get(keySelector
+					.getKey(nextRecord.getObject()));
+			if (groupInvokable == null) {
+				groupInvokable = makeNewGroup(nextRecord);
+			}
+
+			// Run the precalls for central active triggers
+			for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
+				Object[] result = trigger.preNotifyTrigger(nextRecord.getObject());
+				for (Object in : result) {
+
+					// If central eviction is used, handle it here
+					if (!activeCentralEvictionPolicies.isEmpty()) {
+						evictElements(centralActiveEviction(in));
+					}
+
+					// process in groups
+					for (StreamDiscretizer<IN> group : windowingGroups.values()) {
+						group.processFakeElement(in, trigger);
+						checkForEmptyGroupBuffer(group);
+					}
+				}
+			}
+
+			// Process non-active central triggers
+			for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
+				if (triggerPolicy.notifyTrigger(nextRecord.getObject())) {
+					currentTriggerPolicies.add(triggerPolicy);
+				}
+			}
+
+			if (currentTriggerPolicies.isEmpty()) {
+
+				// only add the element to its group
+				groupInvokable.processRealElement(nextRecord.getObject());
+				checkForEmptyGroupBuffer(groupInvokable);
+
+				// If central eviction is used, handle it here
+				if (!centralEvictionPolicies.isEmpty()) {
+					evictElements(centralEviction(nextRecord.getObject(), false));
+					deleteOrderForCentralEviction.add(groupInvokable);
+				}
+
+			} else {
+
+				// call user function for all groups
+				for (StreamDiscretizer<IN> group : windowingGroups.values()) {
+					if (group == groupInvokable) {
+						// process real with initialized policies
+						group.processRealElement(nextRecord.getObject(), currentTriggerPolicies);
+					} else {
+						// process like a fake but also initialized with
+						// policies
+						group.externalTriggerFakeElement(nextRecord.getObject(),
+								currentTriggerPolicies);
+					}
+
+					// remove group in case it has an empty buffer
+					// checkForEmptyGroupBuffer(group);
+				}
+
+				// If central eviction is used, handle it here
+				if (!centralEvictionPolicies.isEmpty()) {
+					evictElements(centralEviction(nextRecord.getObject(), true));
+					deleteOrderForCentralEviction.add(groupInvokable);
+				}
+			}
+
+			// clear current trigger list
+			currentTriggerPolicies.clear();
+
+			// read next record
+			readNext();
+		}
+
+		// Stop all remaining threads from policies
+		for (Thread t : activePolicyThreads) {
+			t.interrupt();
+		}
+
+		// finally trigger the buffer.
+		for (StreamDiscretizer<IN> group : windowingGroups.values()) {
+			group.emitFinalWindow(centralTriggerPolicies);
+		}
+
+	}
+
+	/**
+	 * This method creates a new group. The method gets called in case an
+	 * element arrives which has a key which was not seen before. The method
+	 * created a nested {@link WindowInvokable} and therefore created clones of
+	 * all distributed trigger and eviction policies.
+	 * 
+	 * @param element
+	 *            The element which leads to the generation of a new group
+	 *            (previously unseen key)
+	 * @throws Exception
+	 *             In case the {@link KeySelector} throws an exception in
+	 *             {@link KeySelector#getKey(Object)}, the exception is not
+	 *             catched by this method.
+	 */
+	private StreamDiscretizer<IN> makeNewGroup(StreamRecord<IN> element) throws Exception {
+		// clone the policies
+		LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+		LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
+		for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
+			clonedDistributedTriggerPolicies.add(trigger.clone());
+		}
+		for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
+			clonedDistributedEvictionPolicies.add(eviction.clone());
+		}
+
+		StreamDiscretizer<IN> groupDiscretizer = new StreamDiscretizer<IN>(
+				clonedDistributedTriggerPolicies, clonedDistributedEvictionPolicies);
+
+		groupDiscretizer.setup(taskContext);
+		groupDiscretizer.open(this.parameters);
+		windowingGroups.put(keySelector.getKey(element.getObject()), groupDiscretizer);
+
+		return groupDiscretizer;
+	}
+
+	@Override
+	public void open(Configuration parameters) throws Exception {
+		super.open(parameters);
+		this.parameters = parameters;
+		for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
+			Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+			if (target != null) {
+				Thread thread = new Thread(target);
+				activePolicyThreads.add(thread);
+				thread.start();
+			}
+		}
+	};
+
+	/**
+	 * This method is used to notify central eviction policies with a real
+	 * element.
+	 * 
+	 * @param input
+	 *            the real element to notify the eviction policy.
+	 * @param triggered
+	 *            whether a central trigger occurred or not.
+	 * @return The number of elements to be deleted from the buffer.
+	 */
+	private int centralEviction(IN input, boolean triggered) {
+		// Process the evictions and take care of double evictions
+		// In case there are multiple eviction policies present,
+		// only the one with the highest return value is recognized.
+		int currentMaxEviction = 0;
+		for (EvictionPolicy<IN> evictionPolicy : centralEvictionPolicies) {
+			// use temporary variable to prevent multiple calls to
+			// notifyEviction
+			int tmp = evictionPolicy.notifyEviction(input, triggered,
+					deleteOrderForCentralEviction.size());
+			if (tmp > currentMaxEviction) {
+				currentMaxEviction = tmp;
+			}
+		}
+		return currentMaxEviction;
+	}
+
+	/**
+	 * This method is used to notify active central eviction policies with a
+	 * fake element.
+	 * 
+	 * @param input
+	 *            the fake element to notify the active central eviction
+	 *            policies.
+	 * @return The number of elements to be deleted from the buffer.
+	 */
+	private int centralActiveEviction(Object input) {
+		// Process the evictions and take care of double evictions
+		// In case there are multiple eviction policies present,
+		// only the one with the highest return value is recognized.
+		int currentMaxEviction = 0;
+		for (ActiveEvictionPolicy<IN> evictionPolicy : activeCentralEvictionPolicies) {
+			// use temporary variable to prevent multiple calls to
+			// notifyEviction
+			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input,
+					deleteOrderForCentralEviction.size());
+			if (tmp > currentMaxEviction) {
+				currentMaxEviction = tmp;
+			}
+		}
+		return currentMaxEviction;
+	}
+
+	/**
+	 * This method is used in central eviction to delete a given number of
+	 * elements from the buffer.
+	 * 
+	 * @param numToEvict
+	 *            number of elements to delete from the virtual central element
+	 *            buffer.
+	 */
+	private void evictElements(int numToEvict) {
+		HashSet<StreamDiscretizer<IN>> usedGroups = new HashSet<StreamDiscretizer<IN>>();
+		for (; numToEvict > 0; numToEvict--) {
+			StreamDiscretizer<IN> currentGroup = deleteOrderForCentralEviction.getFirst();
+			// Do the eviction
+			currentGroup.evictFirst();
+			// Remember groups which possibly have an empty buffer after the
+			// eviction
+			usedGroups.add(currentGroup);
+			try {
+				deleteOrderForCentralEviction.removeFirst();
+			} catch (NoSuchElementException e) {
+				// when buffer is empty, ignore exception and stop deleting
+				break;
+			}
+
+		}
+
+		// Remove groups with empty buffer
+		for (StreamDiscretizer<IN> group : usedGroups) {
+			checkForEmptyGroupBuffer(group);
+		}
+	}
+
+	/**
+	 * Checks if the element buffer of a given windowing group is empty. If so,
+	 * the group will be deleted.
+	 * 
+	 * @param group
+	 *            The windowing group to be checked and and removed in case its
+	 *            buffer is empty.
+	 */
+	private void checkForEmptyGroupBuffer(StreamDiscretizer<IN> group) {
+		if (group.isBufferEmpty()) {
+			windowingGroups.remove(group);
+		}
+	}
+
+	/**
+	 * This callback class allows to handle the the callbacks done by threads
+	 * defined in active trigger policies
+	 * 
+	 * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
+	 */
+	private class WindowingCallback implements ActiveTriggerCallback {
+		private ActiveTriggerPolicy<IN> policy;
+
+		public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+			this.policy = policy;
+		}
+
+		@Override
+		public void sendFakeElement(Object datapoint) {
+
+			// If central eviction is used, handle it here
+			if (!centralEvictionPolicies.isEmpty()) {
+				evictElements(centralActiveEviction(datapoint));
+			}
+
+			// handle element in groups
+			for (StreamDiscretizer<IN> group : windowingGroups.values()) {
+				group.processFakeElement(datapoint, policy);
+				checkForEmptyGroupBuffer(group);
+			}
+		}
+
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/1146f64c/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
new file mode 100644
index 0000000..a63aee6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
+import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+
+public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>> {
+
+	/**
+	 * Auto-generated serial version UID
+	 */
+	private static final long serialVersionUID = -8038984294071650730L;
+
+	private LinkedList<TriggerPolicy<IN>> triggerPolicies;
+	private LinkedList<EvictionPolicy<IN>> evictionPolicies;
+	private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
+	private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
+	private LinkedList<Thread> activePolicyTreads;
+	protected LinkedList<IN> buffer;
+	private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
+
+	/**
+	 * This constructor created a windowing invokable using trigger and eviction
+	 * policies.
+	 * 
+	 * @param userFunction
+	 *            The user defined {@link ReduceFunction}
+	 * @param triggerPolicies
+	 *            A list of {@link TriggerPolicy}s and/or
+	 *            {@link ActiveTriggerPolicy}s
+	 * @param evictionPolicies
+	 *            A list of {@link EvictionPolicy}s and/or
+	 *            {@link ActiveEvictionPolicy}s
+	 */
+	public StreamDiscretizer(LinkedList<TriggerPolicy<IN>> triggerPolicies,
+			LinkedList<EvictionPolicy<IN>> evictionPolicies) {
+		super(null);
+
+		this.triggerPolicies = triggerPolicies;
+		this.evictionPolicies = evictionPolicies;
+
+		activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
+		for (TriggerPolicy<IN> tp : triggerPolicies) {
+			if (tp instanceof ActiveTriggerPolicy) {
+				activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
+			}
+		}
+
+		activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
+		for (EvictionPolicy<IN> ep : evictionPolicies) {
+			if (ep instanceof ActiveEvictionPolicy) {
+				activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
+			}
+		}
+
+		this.activePolicyTreads = new LinkedList<Thread>();
+		this.buffer = new LinkedList<IN>();
+		this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
+	}
+
+	@Override
+	public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
+		super.open(parameters);
+		for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
+			Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
+			if (target != null) {
+				Thread thread = new Thread(target);
+				activePolicyTreads.add(thread);
+				thread.start();
+			}
+		}
+	}
+
+	/**
+	 * This class allows the active trigger threads to call back and push fake
+	 * elements at any time.
+	 */
+	private class WindowingCallback implements ActiveTriggerCallback {
+		private ActiveTriggerPolicy<IN> policy;
+
+		public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
+			this.policy = policy;
+		}
+
+		@Override
+		public void sendFakeElement(Object datapoint) {
+			processFakeElement(datapoint, this.policy);
+		}
+
+	}
+
+	@Override
+	public void invoke() throws Exception {
+
+		// Prevent empty data streams
+		if (readNext() == null) {
+			throw new RuntimeException("DataStream must not be empty");
+		}
+
+		// Continuously run
+		while (nextRecord != null) {
+			processRealElement(nextRecord.getObject());
+
+			// Load next StreamRecord
+			readNext();
+		}
+
+		// Stop all remaining threads from policies
+		for (Thread t : activePolicyTreads) {
+			t.interrupt();
+		}
+
+		// finally trigger the buffer.
+		emitFinalWindow(null);
+
+	}
+
+	/**
+	 * This method gets called in case of an grouped windowing in case central
+	 * trigger occurred and the arriving element causing the trigger is not part
+	 * of this group.
+	 * 
+	 * Remark: This is NOT the same as
+	 * {@link StreamDiscretizer#processFakeElement(Object, TriggerPolicy)}! Here
+	 * the eviction using active policies takes place after the call to the UDF.
+	 * Usually it is done before when fake elements get submitted. This special
+	 * behaviour is needed to allow the {@link GroupedStreamDiscretizer} to send
+	 * central triggers to all groups, even if the current element does not
+	 * belong to the group.
+	 * 
+	 * @param input
+	 *            a fake input element
+	 * @param policies
+	 *            the list of policies which caused the call with this fake
+	 *            element
+	 */
+	protected synchronized void externalTriggerFakeElement(IN input,
+			List<TriggerPolicy<IN>> policies) {
+
+		// Set the current triggers
+		currentTriggerPolicies.addAll(policies);
+
+		// emit
+		emitWindow();
+
+		// clear the flag collection
+		currentTriggerPolicies.clear();
+
+		// Process the evictions and take care of double evictions
+		// In case there are multiple eviction policies present,
+		// only the one with the highest return value is recognized.
+		int currentMaxEviction = 0;
+		for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+			// use temporary variable to prevent multiple calls to
+			// notifyEviction
+			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+			if (tmp > currentMaxEviction) {
+				currentMaxEviction = tmp;
+			}
+		}
+
+		for (int i = 0; i < currentMaxEviction; i++) {
+			try {
+				buffer.removeFirst();
+			} catch (NoSuchElementException e) {
+				// In case no more elements are in the buffer:
+				// Prevent failure and stop deleting.
+				break;
+			}
+		}
+	}
+
+	/**
+	 * This method processed an arrived fake element The method is synchronized
+	 * to ensure that it cannot interleave with
+	 * {@link StreamDiscretizer#processRealElement(Object)}
+	 * 
+	 * @param input
+	 *            a fake input element
+	 * @param currentPolicy
+	 *            the policy which produced this fake element
+	 */
+	protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
+
+		// Process the evictions and take care of double evictions
+		// In case there are multiple eviction policies present,
+		// only the one with the highest return value is recognized.
+		int currentMaxEviction = 0;
+		for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
+			// use temporary variable to prevent multiple calls to
+			// notifyEviction
+			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
+			if (tmp > currentMaxEviction) {
+				currentMaxEviction = tmp;
+			}
+		}
+
+		for (int i = 0; i < currentMaxEviction; i++) {
+			try {
+				buffer.removeFirst();
+			} catch (NoSuchElementException e) {
+				// In case no more elements are in the buffer:
+				// Prevent failure and stop deleting.
+				break;
+			}
+		}
+
+		// Set the current trigger
+		currentTriggerPolicies.add(currentPolicy);
+
+		// emit
+		emitWindow();
+
+		// clear the flag collection
+		currentTriggerPolicies.clear();
+	}
+
+	/**
+	 * This method processed an arrived real element The method is synchronized
+	 * to ensure that it cannot interleave with
+	 * {@link StreamDiscretizer#processFakeElement(Object)}.
+	 * 
+	 * @param input
+	 *            a real input element
+	 * @param triggerPolicies
+	 *            Allows to set trigger policies which are maintained
+	 *            externally. This is the case for central policies in
+	 *            {@link GroupedStreamDiscretizer}.
+	 */
+	protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
+		this.currentTriggerPolicies.addAll(triggerPolicies);
+		processRealElement(input);
+	}
+
+	/**
+	 * This method processed an arrived real element The method is synchronized
+	 * to ensure that it cannot interleave with
+	 * {@link StreamDiscretizer#processFakeElement(Object)}
+	 * 
+	 * @param input
+	 *            a real input element
+	 */
+	protected synchronized void processRealElement(IN input) {
+
+		// Run the precalls to detect missed windows
+		for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
+			// Remark: In case multiple active triggers are present the ordering
+			// of the different fake elements returned by this triggers becomes
+			// a problem. This might lead to unexpected results...
+			// Should we limit the number of active triggers to 0 or 1?
+			Object[] result = trigger.preNotifyTrigger(input);
+			for (Object in : result) {
+				processFakeElement(in, trigger);
+			}
+		}
+
+		// Remember if a trigger occurred
+		boolean isTriggered = false;
+
+		// Process the triggers
+		for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
+			if (triggerPolicy.notifyTrigger(input)) {
+				currentTriggerPolicies.add(triggerPolicy);
+			}
+		}
+
+		// call user function
+		if (!currentTriggerPolicies.isEmpty()) {
+			// emit
+			emitWindow();
+
+			// clear the flag collection
+			currentTriggerPolicies.clear();
+
+			// remember trigger
+			isTriggered = true;
+		}
+
+		// Process the evictions and take care of double evictions
+		// In case there are multiple eviction policies present,
+		// only the one with the highest return value is recognized.
+		int currentMaxEviction = 0;
+
+		for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
+			// use temporary variable to prevent multiple calls to
+			// notifyEviction
+			int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
+			if (tmp > currentMaxEviction) {
+				currentMaxEviction = tmp;
+			}
+		}
+
+		for (int i = 0; i < currentMaxEviction; i++) {
+			try {
+				buffer.removeFirst();
+			} catch (NoSuchElementException e) {
+				// In case no more elements are in the buffer:
+				// Prevent failure and stop deleting.
+				break;
+			}
+		}
+
+		// Add the current element to the buffer
+		buffer.add(input);
+
+	}
+
+	private void emitWindow() {
+		StreamWindow<IN> currentWindow = new StreamWindow<IN>();
+		currentWindow.addAll(buffer);
+		collector.collect(currentWindow);
+	}
+
+	/**
+	 * This method removes the first element from the element buffer. It is used
+	 * to provide central evictions in {@link GroupedStreamDiscretizer}
+	 */
+	protected synchronized void evictFirst() {
+		try {
+			buffer.removeFirst();
+		} catch (NoSuchElementException e) {
+			// ignore exception
+		}
+	}
+
+	/**
+	 * This method returns whether the element buffer is empty or not. It is
+	 * used to figure out if a group can be deleted or not when
+	 * {@link GroupedStreamDiscretizer} is used.
+	 * 
+	 * @return true in case the buffer is empty otherwise false.
+	 */
+	protected boolean isBufferEmpty() {
+		return buffer.isEmpty();
+	}
+
+	/**
+	 * This method does the final reduce at the end of the stream and emits the
+	 * result.
+	 * 
+	 * @param centralTriggerPolicies
+	 *            Allows to set trigger policies which are maintained
+	 *            externally. This is the case for central policies in
+	 *            {@link GroupedStreamDiscretizer}.
+	 */
+	protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
+		if (!buffer.isEmpty()) {
+			currentTriggerPolicies.clear();
+
+			if (centralTriggerPolicies != null) {
+				currentTriggerPolicies.addAll(centralTriggerPolicies);
+			}
+
+			for (TriggerPolicy<IN> policy : triggerPolicies) {
+				currentTriggerPolicies.add(policy);
+			}
+
+			emitWindow();
+		}
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/1146f64c/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
new file mode 100644
index 0000000..9f223c3
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowFlattener.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class WindowFlattener<T> extends ChainableInvokable<StreamWindow<T>, T> {
+
+	public WindowFlattener() {
+		super(null);
+	}
+
+	private static final long serialVersionUID = 1L;
+
+	@Override
+	public void invoke() throws Exception {
+		while (readNext() != null) {
+			callUserFunctionAndLogException();
+		}
+	}
+
+	@Override
+	protected void callUserFunction() throws Exception {
+		for (T element : nextObject) {
+			collector.collect(element);
+		}
+	}
+
+	@Override
+	public void collect(StreamWindow<T> record) {
+		callUserFunctionAndLogException();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/1146f64c/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
new file mode 100644
index 0000000..23aaf32
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMapper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
+
+public class WindowMapper<IN, OUT> extends MapInvokable<StreamWindow<IN>, StreamWindow<OUT>> {
+
+	private static final long serialVersionUID = 1L;
+
+	GroupReduceFunction<IN, OUT> reducer;
+
+	public WindowMapper(GroupReduceFunction<IN, OUT> reducer) {
+		super(new WindowMapfunction<IN, OUT>(reducer));
+		this.reducer = reducer;
+	}
+
+	private static class WindowMapfunction<T, R> implements
+			MapFunction<StreamWindow<T>, StreamWindow<R>> {
+
+		private static final long serialVersionUID = 1L;
+		GroupReduceFunction<T, R> reducer;
+
+		public WindowMapfunction(GroupReduceFunction<T, R> reducer) {
+			this.reducer = reducer;
+		}
+
+		@Override
+		public StreamWindow<R> map(StreamWindow<T> window) throws Exception {
+			StreamWindow<R> outputWindow = new StreamWindow<R>(window.windowID);
+			outputWindow.numberOfParts = window.numberOfParts;
+
+			reducer.reduce(window, outputWindow);
+
+			return outputWindow;
+		}
+
+	}
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/1146f64c/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
new file mode 100644
index 0000000..738c78f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class WindowCombiner<T> extends ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
+
+	private Map<Integer, StreamWindow<T>> windows;
+
+	public WindowCombiner() {
+		super(null);
+		this.windows = new HashMap<Integer, StreamWindow<T>>();
+	}
+
+	private static final long serialVersionUID = 1L;
+
+	@Override
+	public void invoke() throws Exception {
+		while (readNext() != null) {
+			callUserFunctionAndLogException();
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	protected void callUserFunction() throws Exception {
+		StreamWindow<T> nextWindow = nextObject;
+
+		StreamWindow<T> current = windows.get(nextWindow.windowID);
+
+		if (current == null) {
+			current = nextWindow;
+		} else {
+			current = StreamWindow.merge(current, nextWindow);
+		}
+
+		if (current.numberOfParts == 1) {
+			collector.collect(current);
+			windows.remove(nextWindow.windowID);
+		} else {
+			windows.put(nextWindow.windowID, current);
+		}
+	}
+
+	@Override
+	public void collect(StreamWindow<T> record) {
+		nextObject = record;
+		callUserFunctionAndLogException();
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/1146f64c/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
new file mode 100644
index 0000000..e10692b
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowPartitioner.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.invokable.ChainableInvokable;
+
+public class WindowPartitioner<T> extends
+		ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
+
+	private KeySelector<T, ?> keySelector;
+	private int numberOfSplits;
+
+	public WindowPartitioner(KeySelector<T, ?> keySelector) {
+		super(null);
+		this.keySelector = keySelector;
+	}
+
+	public WindowPartitioner(int numberOfSplits) {
+		super(null);
+		this.numberOfSplits = numberOfSplits;
+	}
+
+	private static final long serialVersionUID = 1L;
+
+	@Override
+	public void invoke() throws Exception {
+		while (readNext() != null) {
+			callUserFunctionAndLogException();
+		}
+	}
+
+	@Override
+	protected void callUserFunction() throws Exception {
+		StreamWindow<T> currentWindow = nextObject;
+		if (keySelector == null) {
+			if (numberOfSplits <= 1) {
+				collector.collect(currentWindow);
+			} else {
+				for (StreamWindow<T> window : currentWindow.split(numberOfSplits)) {
+					collector.collect(window);
+				}
+			}
+		} else {
+
+			for (StreamWindow<T> window : currentWindow.partitionBy(keySelector)) {
+				collector.collect(window);
+			}
+
+		}
+	}
+
+	@Override
+	public void collect(StreamWindow<T> record) {
+		nextObject = record;
+		callUserFunctionAndLogException();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/1146f64c/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
new file mode 100644
index 0000000..b4f965f
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowReducer.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.invokable.operator.MapInvokable;
+
+public class WindowReducer<IN> extends MapInvokable<StreamWindow<IN>, StreamWindow<IN>> {
+
+	private static final long serialVersionUID = 1L;
+
+	ReduceFunction<IN> reducer;
+
+	public WindowReducer(ReduceFunction<IN> reducer) {
+		super(new WindowReduceFunction<IN>(reducer));
+		this.reducer = reducer;
+	}
+
+	private static class WindowReduceFunction<T> implements
+			MapFunction<StreamWindow<T>, StreamWindow<T>> {
+
+		private static final long serialVersionUID = 1L;
+		ReduceFunction<T> reducer;
+
+		public WindowReduceFunction(ReduceFunction<T> reducer) {
+			this.reducer = reducer;
+		}
+
+		@Override
+		public StreamWindow<T> map(StreamWindow<T> window) throws Exception {
+			StreamWindow<T> outputWindow = new StreamWindow<T>(window.windowID);
+			outputWindow.numberOfParts = window.numberOfParts;
+
+			if (!window.isEmpty()) {
+				T reduced = window.get(0);
+				for (int i = 1; i < window.size(); i++) {
+					reduced = reducer.reduce(reduced, window.get(i));
+				}
+				outputWindow.add(reduced);
+			}
+			return outputWindow;
+		}
+
+	}
+
+}


[13/20] flink git commit: [streaming] GroupedTimeDiscretizer added for lighter time policy thread management

Posted by mb...@apache.org.
[streaming] GroupedTimeDiscretizer added for lighter time policy thread management


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/87086882
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/87086882
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/87086882

Branch: refs/heads/master
Commit: 870868828e7329c03547520bdb1762b4a1763514
Parents: aef52e8
Author: Gyula Fora <gy...@apache.org>
Authored: Sun Feb 8 22:59:22 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:08 2015 +0100

----------------------------------------------------------------------
 .../api/datastream/WindowedDataStream.java      |  32 ++++--
 .../windowing/GroupedStreamDiscretizer.java     |  12 +--
 .../windowing/GroupedTimeDiscretizer.java       | 101 +++++++++++++++++++
 .../operator/windowing/StreamDiscretizer.java   |  22 +++-
 .../api/windowing/policy/TimeTriggerPolicy.java |  12 ++-
 .../scala/examples/join/WindowJoin.scala        |  19 +++-
 6 files changed, 167 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/87086882/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 7c2dc47..7a214fe 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -32,6 +32,7 @@ import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
 import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
 import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedStreamDiscretizer;
+import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedTimeDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamDiscretizer;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
 import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
@@ -40,6 +41,7 @@ import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
 import org.apache.flink.streaming.util.keys.KeySelectorUtil;
@@ -210,15 +212,7 @@ public class WindowedDataStream<OUT> {
 
 	private DiscretizedStream<OUT> discretize(boolean isMap) {
 
-		StreamInvokable<OUT, StreamWindow<OUT>> discretizer;
-
-		if (discretizerKey == null) {
-			discretizer = new StreamDiscretizer<OUT>(getTrigger(), getEvicter());
-		} else {
-			discretizer = new GroupedStreamDiscretizer<OUT>(discretizerKey,
-					(CloneableTriggerPolicy<OUT>) getTrigger(),
-					(CloneableEvictionPolicy<OUT>) getEvicter());
-		}
+		StreamInvokable<OUT, StreamWindow<OUT>> discretizer = getDiscretizer();
 
 		int parallelism = isLocal || (discretizerKey != null) ? dataStream.environment
 				.getDegreeOfParallelism() : 1;
@@ -229,6 +223,22 @@ public class WindowedDataStream<OUT> {
 
 	}
 
+	private StreamInvokable<OUT, StreamWindow<OUT>> getDiscretizer() {
+		if (discretizerKey == null) {
+			return new StreamDiscretizer<OUT>(getTrigger(), getEvicter());
+		} else if (getTrigger() instanceof TimeTriggerPolicy
+				&& ((TimeTriggerPolicy<OUT>) getTrigger()).timestampWrapper.isDefaultTimestamp()) {
+			return new GroupedTimeDiscretizer<OUT>(discretizerKey,
+					(TimeTriggerPolicy<OUT>) getTrigger(),
+					(CloneableEvictionPolicy<OUT>) getEvicter());
+		} else {
+			return new GroupedStreamDiscretizer<OUT>(discretizerKey,
+					(CloneableTriggerPolicy<OUT>) getTrigger(),
+					(CloneableEvictionPolicy<OUT>) getEvicter());
+		}
+
+	}
+
 	/**
 	 * Applies a reduce transformation on the windowed data stream by reducing
 	 * the current window at every trigger.The user can also extend the
@@ -276,8 +286,8 @@ public class WindowedDataStream<OUT> {
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
-	public <R> WindowedDataStream<R> mapWindow(
-			GroupReduceFunction<OUT, R> reduceFunction, TypeInformation<R> outType) {
+	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction,
+			TypeInformation<R> outType) {
 
 		return discretize(true).mapWindow(reduceFunction, outType);
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/87086882/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
index efd2e06..e90726d 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizer.java
@@ -33,12 +33,12 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 	 */
 	private static final long serialVersionUID = -3469545957144404137L;
 
-	private KeySelector<IN, ?> keySelector;
-	private Configuration parameters;
-	private CloneableTriggerPolicy<IN> triggerPolicy;
-	private CloneableEvictionPolicy<IN> evictionPolicy;
+	protected KeySelector<IN, ?> keySelector;
+	protected Configuration parameters;
+	protected CloneableTriggerPolicy<IN> triggerPolicy;
+	protected CloneableEvictionPolicy<IN> evictionPolicy;
 
-	private Map<Object, StreamDiscretizer<IN>> groupedDiscretizers;
+	protected Map<Object, StreamDiscretizer<IN>> groupedDiscretizers;
 
 	public GroupedStreamDiscretizer(KeySelector<IN, ?> keySelector,
 			CloneableTriggerPolicy<IN> triggerPolicy, CloneableEvictionPolicy<IN> evictionPolicy) {
@@ -92,7 +92,7 @@ public class GroupedStreamDiscretizer<IN> extends StreamInvokable<IN, StreamWind
 	 * @param key
 	 *            The key of the new group.
 	 */
-	private StreamDiscretizer<IN> makeNewGroup(Object key) throws Exception {
+	protected StreamDiscretizer<IN> makeNewGroup(Object key) throws Exception {
 
 		StreamDiscretizer<IN> groupDiscretizer = new StreamDiscretizer<IN>(triggerPolicy.clone(),
 				evictionPolicy.clone());

http://git-wip-us.apache.org/repos/asf/flink/blob/87086882/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
new file mode 100644
index 0000000..63901a8
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedTimeDiscretizer.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+
+public class GroupedTimeDiscretizer<IN> extends GroupedStreamDiscretizer<IN> {
+
+	private static final long serialVersionUID = -3469545957144404137L;
+
+	private TimeTriggerPolicy<IN> timeTriggerPolicy;
+	private Thread policyThread;
+
+	public GroupedTimeDiscretizer(KeySelector<IN, ?> keySelector,
+			TimeTriggerPolicy<IN> triggerPolicy, CloneableEvictionPolicy<IN> evictionPolicy) {
+
+		super(keySelector, triggerPolicy, evictionPolicy);
+		this.timeTriggerPolicy = triggerPolicy;
+	}
+
+	@Override
+	protected StreamDiscretizer<IN> makeNewGroup(Object key) throws Exception {
+
+		StreamDiscretizer<IN> groupDiscretizer = new StreamDiscretizer<IN>(triggerPolicy.clone(),
+				evictionPolicy.clone());
+
+		groupDiscretizer.collector = taskContext.getOutputCollector();
+		// We omit the groupDiscretizer.open(...) call here to avoid starting
+		// new active threads
+		return groupDiscretizer;
+	}
+
+	@Override
+	public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
+		super.open(parameters);
+
+		Runnable runnable = new TimeCheck();
+		policyThread = new Thread(runnable);
+		policyThread.start();
+	}
+
+	private void removeUnusedGroups(int threshold) {
+		List<Object> toRemove = new ArrayList<Object>();
+
+		for (Entry<Object, StreamDiscretizer<IN>> entry : groupedDiscretizers.entrySet()) {
+			if (entry.getValue().emptyCount > threshold) {
+				toRemove.add(entry.getKey());
+			}
+		}
+
+		for (Object key : toRemove) {
+			groupedDiscretizers.remove(key);
+		}
+	}
+
+	private class TimeCheck implements Runnable {
+
+		@Override
+		public void run() {
+			while (true) {
+				// wait for the specified granularity
+				try {
+					Thread.sleep(timeTriggerPolicy.granularity);
+				} catch (InterruptedException e) {
+					// ignore it...
+				}
+
+				for (StreamDiscretizer<IN> group : groupedDiscretizers.values()) {
+					TimeTriggerPolicy<IN> groupTrigger = (TimeTriggerPolicy<IN>) group.triggerPolicy;
+					Object fake = groupTrigger.activeFakeElementEmission(null);
+					if (fake != null) {
+						group.triggerOnFakeElement(fake);
+					}
+				}
+
+				removeUnusedGroups(10);
+			}
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/87086882/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
index d5b4354..ac02af3 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizer.java
@@ -34,12 +34,13 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 	 */
 	private static final long serialVersionUID = -8038984294071650730L;
 
-	private TriggerPolicy<IN> triggerPolicy;
-	private EvictionPolicy<IN> evictionPolicy;
+	protected TriggerPolicy<IN> triggerPolicy;
+	protected EvictionPolicy<IN> evictionPolicy;
 	private boolean isActiveTrigger;
 	private boolean isActiveEviction;
 	private Thread activePolicyThread;
 	protected LinkedList<IN> buffer;
+	public int emptyCount = 0;
 
 	public StreamDiscretizer(TriggerPolicy<IN> triggerPolicy, EvictionPolicy<IN> evictionPolicy) {
 		super(null);
@@ -113,13 +114,24 @@ public class StreamDiscretizer<IN> extends StreamInvokable<IN, StreamWindow<IN>>
 		emitWindow();
 	}
 
+	protected synchronized void externalTriggerOnFakeElement(Object input) {
+		emitWindow();
+		activeEvict(input);
+	}
+
 	/**
 	 * This method emits the content of the buffer as a new {@link StreamWindow}
+	 * if not empty
 	 */
 	protected void emitWindow() {
-		StreamWindow<IN> currentWindow = new StreamWindow<IN>();
-		currentWindow.addAll(buffer);
-		collector.collect(currentWindow);
+		if (!buffer.isEmpty()) {
+			StreamWindow<IN> currentWindow = new StreamWindow<IN>();
+			currentWindow.addAll(buffer);
+			collector.collect(currentWindow);
+			emptyCount = 0;
+		} else {
+			emptyCount++;
+		}
 	}
 
 	private void activeEvict(Object input) {

http://git-wip-us.apache.org/repos/asf/flink/blob/87086882/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
index 1e91b8e..fb249cf 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/TimeTriggerPolicy.java
@@ -41,8 +41,8 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
 	private static final long serialVersionUID = -5122753802440196719L;
 
 	protected long startTime;
-	protected long granularity;
-	protected TimestampWrapper<DATA> timestampWrapper;
+	public long granularity;
+	public TimestampWrapper<DATA> timestampWrapper;
 	protected long delay;
 
 	/**
@@ -141,13 +141,17 @@ public class TimeTriggerPolicy<DATA> implements ActiveTriggerPolicy<DATA>,
 	 * @param callback
 	 *            The callback object.
 	 */
-	private synchronized void activeFakeElementEmission(ActiveTriggerCallback callback) {
+	public synchronized Object activeFakeElementEmission(ActiveTriggerCallback callback) {
 
 		// start time is excluded, but end time is included: >=
 		if (System.currentTimeMillis() >= startTime + granularity) {
 			startTime += granularity;
-			callback.sendFakeElement(startTime - 1);
+			if (callback != null) {
+				callback.sendFakeElement(startTime - 1);
+			}
+			return startTime - 1;
 		}
+		return null;
 
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/87086882/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
index 6ea7be0..abd6d50 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
@@ -34,12 +34,21 @@ object WindowJoin {
 
   def main(args: Array[String]) {
 
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.setDegreeOfParallelism(1)
+   val env = StreamExecutionEnvironment.getExecutionEnvironment
 
-    val split = env.generateSequence(1, 10).split(x => x % 3 toString)
-    
-    split.select("0").merge(split.select("1")).print
+    //Create streams for names and ages by mapping the inputs to the corresponding objects
+    val names = env.fromCollection(nameStream).map(x => Name(x._1, x._2))
+    val ages = env.fromCollection(ageStream).map(x => Age(x._1, x._2))
+
+    //Join the two input streams by id on the last 2 seconds every second and create new 
+    //Person objects containing both name and age
+    val joined =
+      names.join(ages).onWindow(2, TimeUnit.SECONDS)
+                      .every(1, TimeUnit.SECONDS)
+                      .where("id")
+                      .equalTo("id") { (n, a) => Person(n.name, a.age) }
+
+    joined print
 
     env.execute("WindowJoin")
   }


[04/20] flink git commit: [FLINK-1176] [streaming] WindowedDataStream rework for new windowing runtime

Posted by mb...@apache.org.
[FLINK-1176] [streaming] WindowedDataStream rework for new windowing runtime


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c560d76f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c560d76f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c560d76f

Branch: refs/heads/master
Commit: c560d76fec7b0e389430fc418d83d8f38f5eb06f
Parents: 1146f64
Author: Gyula Fora <gy...@apache.org>
Authored: Wed Feb 4 17:27:34 2015 +0100
Committer: mbalassi <mb...@apache.org>
Committed: Mon Feb 16 13:06:07 2015 +0100

----------------------------------------------------------------------
 .../api/datastream/DiscretizedStream.java       | 217 +++++++
 .../api/datastream/WindowedDataStream.java      | 266 +++++----
 .../operator/GroupedWindowInvokable.java        | 506 ----------------
 .../operator/WindowGroupReduceInvokable.java    |  51 --
 .../api/invokable/operator/WindowInvokable.java | 382 ------------
 .../operator/WindowReduceInvokable.java         |  59 --
 .../operator/windowing/WindowMerger.java        |   4 +-
 .../policy/CloneableEvictionPolicy.java         |   1 -
 .../policy/CloneableTriggerPolicy.java          |   1 -
 .../operator/GroupedWindowInvokableTest.java    | 574 -------------------
 .../invokable/operator/WindowInvokableTest.java | 261 ---------
 .../windowing/GroupedWindowInvokableTest.java   | 574 +++++++++++++++++++
 .../operator/windowing/WindowInvokableTest.java | 261 +++++++++
 .../ml/IncrementalLearningSkeleton.java         |   2 +-
 .../socket/SocketTextStreamWordCount.java       |  27 +-
 .../windowing/TopSpeedWindowingExample.java     |   2 +-
 .../scala/examples/join/WindowJoin.scala        |  45 +-
 .../examples/windowing/TopSpeedWindowing.scala  |   1 +
 .../api/scala/WindowedDataStream.scala          |  12 +-
 19 files changed, 1238 insertions(+), 2008 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
new file mode 100644
index 0000000..37fa1e1
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/DiscretizedStream.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.datastream;
+
+import org.apache.flink.api.common.functions.GroupReduceFunction;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.functions.RichGroupReduceFunction;
+import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.typeutils.TupleTypeInfo;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.streaming.api.invokable.StreamInvokable;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowFlattener;
+import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
+import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowMerger;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowMapper;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowPartitioner;
+import org.apache.flink.streaming.api.invokable.operator.windowing.WindowReducer;
+
+/**
+ * A {@link DiscretizedStream} represents a data stream that has been divided
+ * into windows (predefined chunks). User defined function such as
+ * {@link #reduceWindow(ReduceFunction)},
+ * {@link #mapWindow(GroupReduceFunction)} or aggregations can be applied to the
+ * windows.
+ * 
+ * @param <OUT>
+ *            The output type of the {@link DiscretizedStream}
+ */
+public class DiscretizedStream<OUT> extends WindowedDataStream<OUT> {
+
+	protected SingleOutputStreamOperator<StreamWindow<OUT>, ?> discretizedStream;
+
+	protected DiscretizedStream(SingleOutputStreamOperator<StreamWindow<OUT>, ?> discretizedStream,
+			KeySelector<OUT, ?> groupByKey) {
+		super();
+		this.groupByKey = groupByKey;
+		this.discretizedStream = discretizedStream;
+	}
+
+	/**
+	 * Applies a reduce transformation on the windowed data stream by reducing
+	 * the current window at every trigger.The user can also extend the
+	 * {@link RichReduceFunction} to gain access to other features provided by
+	 * the {@link org.apache.flink.api.common.functions.RichFunction} interface.
+	 * 
+	 * @param reduceFunction
+	 *            The reduce function that will be applied to the windows.
+	 * @return The transformed DataStream
+	 */
+	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
+
+		DiscretizedStream<OUT> out = partition(false).transform("Window Reduce", getType(),
+				new WindowReducer<OUT>(reduceFunction)).merge();
+
+		if (!isGrouped()) {
+			return out.transform("Window Reduce", out.getType(), new WindowReducer<OUT>(
+					reduceFunction));
+		} else {
+			return out;
+		}
+	}
+
+	/**
+	 * Applies a reduceGroup transformation on the windowed data stream by
+	 * reducing the current window at every trigger. In contrast with the
+	 * standard binary reducer, with reduceGroup the user can access all
+	 * elements of the window at the same time through the iterable interface.
+	 * The user can also extend the {@link RichGroupReduceFunction} to gain
+	 * access to other features provided by the
+	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
+	 * 
+	 * @param reduceFunction
+	 *            The reduce function that will be applied to the windows.
+	 * @return The transformed DataStream
+	 */
+	public <R> DiscretizedStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction) {
+
+		TypeInformation<R> retType = TypeExtractor.getGroupReduceReturnTypes(reduceFunction,
+				getType());
+
+		DiscretizedStream<R> out = partition(true).transform("Window Reduce", retType,
+				new WindowMapper<OUT, R>(reduceFunction));
+
+		if (isGrouped()) {
+			return out.merge();
+		} else {
+			return out;
+		}
+
+	}
+
+	private <R> DiscretizedStream<R> transform(String operatorName, TypeInformation<R> retType,
+			StreamInvokable<StreamWindow<OUT>, StreamWindow<R>> invokable) {
+
+		return wrap(discretizedStream.transform(operatorName, new StreamWindowTypeInfo<R>(retType),
+				invokable));
+	}
+
+	private DiscretizedStream<OUT> partition(boolean isMap) {
+
+		int parallelism = discretizedStream.getParallelism();
+
+		if (isGrouped()) {
+			DiscretizedStream<OUT> out = transform("Window partitioner", getType(),
+					new WindowPartitioner<OUT>(groupByKey)).setParallelism(parallelism);
+
+			out.groupByKey = null;
+
+			return out;
+		} else if (!isMap) {
+			return transform(
+					"Window partitioner",
+					getType(),
+					new WindowPartitioner<OUT>(discretizedStream.environment
+							.getDegreeOfParallelism())).setParallelism(parallelism);
+		} else {
+			return this;
+		}
+	}
+
+	private DiscretizedStream<OUT> setParallelism(int parallelism) {
+		return wrap(discretizedStream.setParallelism(parallelism));
+	}
+
+	private DiscretizedStream<OUT> merge() {
+		TypeInformation<StreamWindow<OUT>> type = discretizedStream.getType();
+
+		return wrap(discretizedStream.groupBy(new WindowKey<OUT>()).transform("Window Merger",
+				type, new WindowMerger<OUT>()));
+	}
+
+	public DataStream<OUT> flatten() {
+		return discretizedStream.transform("Window Flatten", getType(), new WindowFlattener<OUT>());
+	}
+
+	@SuppressWarnings({ "unchecked", "rawtypes" })
+	private <R> DiscretizedStream<R> wrap(SingleOutputStreamOperator stream) {
+		return new DiscretizedStream<R>(stream, (KeySelector<R, ?>) this.groupByKey);
+	}
+
+	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
+		return discretizedStream;
+	}
+
+	@SuppressWarnings("rawtypes")
+	protected Class<?> getClassAtPos(int pos) {
+		Class<?> type;
+		TypeInformation<OUT> outTypeInfo = getType();
+		if (outTypeInfo.isTupleType()) {
+			type = ((TupleTypeInfo) outTypeInfo).getTypeAt(pos).getTypeClass();
+
+		} else if (outTypeInfo instanceof BasicArrayTypeInfo) {
+
+			type = ((BasicArrayTypeInfo) outTypeInfo).getComponentTypeClass();
+
+		} else if (outTypeInfo instanceof PrimitiveArrayTypeInfo) {
+			Class<?> clazz = outTypeInfo.getTypeClass();
+			if (clazz == boolean[].class) {
+				type = Boolean.class;
+			} else if (clazz == short[].class) {
+				type = Short.class;
+			} else if (clazz == int[].class) {
+				type = Integer.class;
+			} else if (clazz == long[].class) {
+				type = Long.class;
+			} else if (clazz == float[].class) {
+				type = Float.class;
+			} else if (clazz == double[].class) {
+				type = Double.class;
+			} else if (clazz == char[].class) {
+				type = Character.class;
+			} else {
+				throw new IndexOutOfBoundsException("Type could not be determined for array");
+			}
+
+		} else if (pos == 0) {
+			type = outTypeInfo.getTypeClass();
+		} else {
+			throw new IndexOutOfBoundsException("Position is out of range");
+		}
+		return type;
+	}
+
+	/**
+	 * Gets the output type.
+	 * 
+	 * @return The output type.
+	 */
+	public TypeInformation<OUT> getType() {
+		return ((StreamWindowTypeInfo<OUT>) discretizedStream.getType()).getInnerType();
+	}
+
+	protected DiscretizedStream<OUT> copy() {
+		return new DiscretizedStream<OUT>(discretizedStream.copy(), groupByKey);
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
index 4fe356b..fc117a1 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/WindowedDataStream.java
@@ -25,17 +25,20 @@ import org.apache.flink.api.common.functions.GroupReduceFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RichGroupReduceFunction;
 import org.apache.flink.api.common.functions.RichReduceFunction;
+import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
+import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.api.java.operators.Keys;
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction;
 import org.apache.flink.streaming.api.function.aggregation.AggregationFunction.AggregationType;
 import org.apache.flink.streaming.api.function.aggregation.ComparableAggregator;
 import org.apache.flink.streaming.api.function.aggregation.SumAggregator;
 import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowGroupReduceInvokable;
-import org.apache.flink.streaming.api.invokable.operator.WindowReduceInvokable;
+import org.apache.flink.streaming.api.invokable.operator.windowing.GroupedStreamDiscretizer;
+import org.apache.flink.streaming.api.invokable.operator.windowing.StreamDiscretizer;
+import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindow;
+import org.apache.flink.streaming.api.invokable.operator.windowing.StreamWindowTypeInfo;
 import org.apache.flink.streaming.api.windowing.helper.Time;
 import org.apache.flink.streaming.api.windowing.helper.WindowingHelper;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
@@ -44,12 +47,14 @@ import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.keys.KeySelectorUtil;
 
 /**
  * A {@link WindowedDataStream} represents a data stream that has been divided
  * into windows (predefined chunks). User defined function such as
- * {@link #reduce(ReduceFunction)}, {@link #reduceGroup(GroupReduceFunction)} or
- * aggregations can be applied to the windows.
+ * {@link #reduceWindow(ReduceFunction)},
+ * {@link #mapWindow(GroupReduceFunction)} or aggregations can be applied to the
+ * windows.
  * 
  * @param <OUT>
  *            The output type of the {@link WindowedDataStream}
@@ -57,9 +62,12 @@ import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
 public class WindowedDataStream<OUT> {
 
 	protected DataStream<OUT> dataStream;
-	protected boolean isGrouped;
-	protected boolean allCentral;
-	protected KeySelector<OUT, ?> keySelector;
+
+	protected boolean isLocal = false;
+	protected boolean isCentral = true;
+
+	protected KeySelector<OUT, ?> discretizerKey;
+	protected KeySelector<OUT, ?> groupByKey;
 
 	protected List<WindowingHelper<OUT>> triggerHelpers;
 	protected List<WindowingHelper<OUT>> evictionHelpers;
@@ -67,6 +75,10 @@ public class WindowedDataStream<OUT> {
 	protected LinkedList<TriggerPolicy<OUT>> userTriggers;
 	protected LinkedList<EvictionPolicy<OUT>> userEvicters;
 
+	protected WindowedDataStream() {
+
+	}
+
 	protected WindowedDataStream(DataStream<OUT> dataStream, WindowingHelper<OUT>... policyHelpers) {
 		this.dataStream = dataStream.copy();
 		this.triggerHelpers = new ArrayList<WindowingHelper<OUT>>();
@@ -75,15 +87,9 @@ public class WindowedDataStream<OUT> {
 		}
 
 		if (dataStream instanceof GroupedDataStream) {
-			this.isGrouped = true;
-			this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+			this.discretizerKey = ((GroupedDataStream<OUT>) dataStream).keySelector;
 			// set all policies distributed
-			this.allCentral = false;
-
-		} else {
-			this.isGrouped = false;
-			// set all policies central
-			this.allCentral = true;
+			this.isCentral = false;
 		}
 	}
 
@@ -102,27 +108,23 @@ public class WindowedDataStream<OUT> {
 		}
 
 		if (dataStream instanceof GroupedDataStream) {
-			this.isGrouped = true;
-			this.keySelector = ((GroupedDataStream<OUT>) dataStream).keySelector;
+			this.discretizerKey = ((GroupedDataStream<OUT>) dataStream).keySelector;
 			// set all policies distributed
-			this.allCentral = false;
+			this.isCentral = false;
 
-		} else {
-			this.isGrouped = false;
-			// set all policies central
-			this.allCentral = true;
 		}
 	}
 
 	protected WindowedDataStream(WindowedDataStream<OUT> windowedDataStream) {
 		this.dataStream = windowedDataStream.dataStream.copy();
-		this.isGrouped = windowedDataStream.isGrouped;
-		this.keySelector = windowedDataStream.keySelector;
+		this.discretizerKey = windowedDataStream.discretizerKey;
+		this.groupByKey = windowedDataStream.groupByKey;
 		this.triggerHelpers = windowedDataStream.triggerHelpers;
 		this.evictionHelpers = windowedDataStream.evictionHelpers;
 		this.userTriggers = windowedDataStream.userTriggers;
 		this.userEvicters = windowedDataStream.userEvicters;
-		this.allCentral = windowedDataStream.allCentral;
+		this.isCentral = windowedDataStream.isCentral;
+		this.isLocal = windowedDataStream.isLocal;
 	}
 
 	public <F> F clean(F f) {
@@ -169,11 +171,11 @@ public class WindowedDataStream<OUT> {
 	 * @return The grouped {@link WindowedDataStream}
 	 */
 	public WindowedDataStream<OUT> groupBy(int... fields) {
-		WindowedDataStream<OUT> ret = this.copy();
-		ret.dataStream = ret.dataStream.groupBy(fields);
-		ret.isGrouped = true;
-		ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
-		return ret;
+		if (getType() instanceof BasicArrayTypeInfo || getType() instanceof PrimitiveArrayTypeInfo) {
+			return groupBy(new KeySelectorUtil.ArrayKeySelector<OUT>(fields));
+		} else {
+			return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
+		}
 	}
 
 	/**
@@ -193,11 +195,7 @@ public class WindowedDataStream<OUT> {
 	 * @return The grouped {@link WindowedDataStream}
 	 */
 	public WindowedDataStream<OUT> groupBy(String... fields) {
-		WindowedDataStream<OUT> ret = this.copy();
-		ret.dataStream = ret.dataStream.groupBy(fields);
-		ret.isGrouped = true;
-		ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
-		return ret;
+		return groupBy(new Keys.ExpressionKeys<OUT>(fields, getType()));
 	}
 
 	/**
@@ -214,12 +212,48 @@ public class WindowedDataStream<OUT> {
 	 */
 	public WindowedDataStream<OUT> groupBy(KeySelector<OUT, ?> keySelector) {
 		WindowedDataStream<OUT> ret = this.copy();
-		ret.dataStream = ret.dataStream.groupBy(keySelector);
-		ret.isGrouped = true;
-		ret.keySelector = ((GroupedDataStream<OUT>) ret.dataStream).keySelector;
+		ret.groupByKey = keySelector;
 		return ret;
 	}
 
+	private WindowedDataStream<OUT> groupBy(Keys<OUT> keys) {
+		return groupBy(clean(KeySelectorUtil.getSelectorForKeys(keys, getType())));
+	}
+
+	/**
+	 * Sets the windowed computations local, so that the windowing and reduce or
+	 * aggregation logic will be computed for each parallel instance of this
+	 * operator
+	 * 
+	 * @return The local windowed data stream
+	 */
+	public WindowedDataStream<OUT> local() {
+		WindowedDataStream<OUT> out = copy();
+		out.isLocal = true;
+		return out;
+	}
+
+	private DiscretizedStream<OUT> discretize(boolean isMap) {
+
+		StreamInvokable<OUT, StreamWindow<OUT>> discretizer;
+
+		if (discretizerKey == null) {
+			discretizer = new StreamDiscretizer<OUT>(getTriggers(), getEvicters());
+		} else {
+			discretizer = new GroupedStreamDiscretizer<OUT>(discretizerKey,
+					getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
+					getCentralEvicters());
+		}
+
+		int parallelism = isLocal || (discretizerKey != null) ? dataStream.environment
+				.getDegreeOfParallelism() : 1;
+
+		return new DiscretizedStream<OUT>(dataStream.transform("Stream Discretizer",
+				new StreamWindowTypeInfo<OUT>(getType()), discretizer).setParallelism(parallelism),
+				groupByKey);
+
+	}
+
 	/**
 	 * Applies a reduce transformation on the windowed data stream by reducing
 	 * the current window at every trigger.The user can also extend the
@@ -230,9 +264,9 @@ public class WindowedDataStream<OUT> {
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
-	public SingleOutputStreamOperator<OUT, ?> reduce(ReduceFunction<OUT> reduceFunction) {
-		return dataStream.transform("Window-Reduce", getType(),
-				getReduceInvokable(reduceFunction));
+	public DiscretizedStream<OUT> reduceWindow(ReduceFunction<OUT> reduceFunction) {
+
+		return discretize(false).reduceWindow(reduceFunction);
 	}
 
 	/**
@@ -248,15 +282,12 @@ public class WindowedDataStream<OUT> {
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
-	public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
-			GroupReduceFunction<OUT, R> reduceFunction) {
-
-		TypeInformation<OUT> inType = getType();
-		TypeInformation<R> outType = TypeExtractor
-				.getGroupReduceReturnTypes(reduceFunction, inType);
+	public <R> WindowedDataStream<R> mapWindow(GroupReduceFunction<OUT, R> reduceFunction) {
+		return discretize(true).mapWindow(reduceFunction);
+	}
 
-		return dataStream.transform("WindowReduce", outType,
-				getReduceGroupInvokable(reduceFunction));
+	public DataStream<OUT> flatten() {
+		return dataStream;
 	}
 
 	/**
@@ -269,18 +300,20 @@ public class WindowedDataStream<OUT> {
 	 * {@link org.apache.flink.api.common.functions.RichFunction} interface.
 	 * </br> </br> This version of reduceGroup uses user supplied
 	 * typeinformation for serializaton. Use this only when the system is unable
-	 * to detect type information using:
-	 * {@link #reduceGroup(GroupReduceFunction)}
+	 * to detect type information using: {@link #mapWindow(GroupReduceFunction)}
 	 * 
 	 * @param reduceFunction
 	 *            The reduce function that will be applied to the windows.
 	 * @return The transformed DataStream
 	 */
-	public <R> SingleOutputStreamOperator<R, ?> reduceGroup(
+	public <R> SingleOutputStreamOperator<R, ?> mapWindow(
 			GroupReduceFunction<OUT, R> reduceFunction, TypeInformation<R> outType) {
 
-		return dataStream.transform("Window-Reduce", outType,
-				getReduceGroupInvokable(reduceFunction));
+		throw new RuntimeException("Not implemented yet");
+	}
+
+	protected Class<?> getClassAtPos(int pos) {
+		return dataStream.getClassAtPos(pos);
 	}
 
 	/**
@@ -291,10 +324,9 @@ public class WindowedDataStream<OUT> {
 	 *            The position in the tuple/array to sum
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> sum(int positionToSum) {
-		dataStream.checkFieldRange(positionToSum);
+	public WindowedDataStream<OUT> sum(int positionToSum) {
 		return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(positionToSum,
-				dataStream.getClassAtPos(positionToSum), getType()));
+				getClassAtPos(positionToSum), getType()));
 	}
 
 	/**
@@ -308,7 +340,7 @@ public class WindowedDataStream<OUT> {
 	 *            The field to sum
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> sum(String field) {
+	public WindowedDataStream<OUT> sum(String field) {
 		return aggregate((AggregationFunction<OUT>) SumAggregator.getSumFunction(field, getType()));
 	}
 
@@ -320,8 +352,7 @@ public class WindowedDataStream<OUT> {
 	 *            The position to minimize
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> min(int positionToMin) {
-		dataStream.checkFieldRange(positionToMin);
+	public WindowedDataStream<OUT> min(int positionToMin) {
 		return aggregate(ComparableAggregator.getAggregator(positionToMin, getType(),
 				AggregationType.MIN));
 	}
@@ -338,7 +369,7 @@ public class WindowedDataStream<OUT> {
 	 *            applied.
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> min(String field) {
+	public WindowedDataStream<OUT> min(String field) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MIN,
 				false));
 	}
@@ -352,7 +383,7 @@ public class WindowedDataStream<OUT> {
 	 *            The position to minimize by
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy) {
+	public WindowedDataStream<OUT> minBy(int positionToMinBy) {
 		return this.minBy(positionToMinBy, true);
 	}
 
@@ -365,7 +396,7 @@ public class WindowedDataStream<OUT> {
 	 *            The position to minimize by
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> minBy(String positionToMinBy) {
+	public WindowedDataStream<OUT> minBy(String positionToMinBy) {
 		return this.minBy(positionToMinBy, true);
 	}
 
@@ -382,8 +413,7 @@ public class WindowedDataStream<OUT> {
 	 *            minimum value, otherwise returns the last
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> minBy(int positionToMinBy, boolean first) {
-		dataStream.checkFieldRange(positionToMinBy);
+	public WindowedDataStream<OUT> minBy(int positionToMinBy, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(positionToMinBy, getType(),
 				AggregationType.MINBY, first));
 	}
@@ -403,7 +433,7 @@ public class WindowedDataStream<OUT> {
 	 *            be returned
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> minBy(String field, boolean first) {
+	public WindowedDataStream<OUT> minBy(String field, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(),
 				AggregationType.MINBY, first));
 	}
@@ -416,8 +446,7 @@ public class WindowedDataStream<OUT> {
 	 *            The position to maximize
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> max(int positionToMax) {
-		dataStream.checkFieldRange(positionToMax);
+	public WindowedDataStream<OUT> max(int positionToMax) {
 		return aggregate(ComparableAggregator.getAggregator(positionToMax, getType(),
 				AggregationType.MAX));
 	}
@@ -434,7 +463,7 @@ public class WindowedDataStream<OUT> {
 	 *            applied.
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> max(String field) {
+	public WindowedDataStream<OUT> max(String field) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(), AggregationType.MAX,
 				false));
 	}
@@ -448,7 +477,7 @@ public class WindowedDataStream<OUT> {
 	 *            The position to maximize by
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy) {
+	public WindowedDataStream<OUT> maxBy(int positionToMaxBy) {
 		return this.maxBy(positionToMaxBy, true);
 	}
 
@@ -461,7 +490,7 @@ public class WindowedDataStream<OUT> {
 	 *            The position to maximize by
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> maxBy(String positionToMaxBy) {
+	public WindowedDataStream<OUT> maxBy(String positionToMaxBy) {
 		return this.maxBy(positionToMaxBy, true);
 	}
 
@@ -478,8 +507,7 @@ public class WindowedDataStream<OUT> {
 	 *            maximum value, otherwise returns the last
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> maxBy(int positionToMaxBy, boolean first) {
-		dataStream.checkFieldRange(positionToMaxBy);
+	public WindowedDataStream<OUT> maxBy(int positionToMaxBy, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(positionToMaxBy, getType(),
 				AggregationType.MAXBY, first));
 	}
@@ -499,21 +527,17 @@ public class WindowedDataStream<OUT> {
 	 *            be returned
 	 * @return The transformed DataStream.
 	 */
-	public SingleOutputStreamOperator<OUT, ?> maxBy(String field, boolean first) {
+	public WindowedDataStream<OUT> maxBy(String field, boolean first) {
 		return aggregate(ComparableAggregator.getAggregator(field, getType(),
 				AggregationType.MAXBY, first));
 	}
 
-	private SingleOutputStreamOperator<OUT, ?> aggregate(AggregationFunction<OUT> aggregator) {
-		StreamInvokable<OUT, OUT> invokable = getReduceInvokable(aggregator);
-
-		SingleOutputStreamOperator<OUT, ?> returnStream = dataStream.transform("Window-Aggregation",
-				getType(), invokable);
+	private WindowedDataStream<OUT> aggregate(AggregationFunction<OUT> aggregator) {
 
-		return returnStream;
+		return reduceWindow(aggregator);
 	}
 
-	private LinkedList<TriggerPolicy<OUT>> getTriggers() {
+	protected LinkedList<TriggerPolicy<OUT>> getTriggers() {
 
 		LinkedList<TriggerPolicy<OUT>> triggers = new LinkedList<TriggerPolicy<OUT>>();
 
@@ -531,7 +555,7 @@ public class WindowedDataStream<OUT> {
 
 	}
 
-	private LinkedList<EvictionPolicy<OUT>> getEvicters() {
+	protected LinkedList<EvictionPolicy<OUT>> getEvicters() {
 
 		LinkedList<EvictionPolicy<OUT>> evicters = new LinkedList<EvictionPolicy<OUT>>();
 
@@ -541,15 +565,15 @@ public class WindowedDataStream<OUT> {
 			}
 		} else {
 			if (userEvicters == null) {
-				boolean notOnlyTime=false;
-				for (WindowingHelper<OUT> helper : triggerHelpers){
-					if (helper instanceof Time<?>){
+				boolean notOnlyTime = false;
+				for (WindowingHelper<OUT> helper : triggerHelpers) {
+					if (helper instanceof Time<?>) {
 						evicters.add(helper.toEvict());
 					} else {
-						notOnlyTime=true;
+						notOnlyTime = true;
 					}
 				}
-				if (notOnlyTime){
+				if (notOnlyTime) {
 					evicters.add(new TumblingEvictionPolicy<OUT>());
 				}
 			}
@@ -562,9 +586,9 @@ public class WindowedDataStream<OUT> {
 		return evicters;
 	}
 
-	private LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
+	protected LinkedList<TriggerPolicy<OUT>> getCentralTriggers() {
 		LinkedList<TriggerPolicy<OUT>> cTriggers = new LinkedList<TriggerPolicy<OUT>>();
-		if (allCentral) {
+		if (isCentral) {
 			cTriggers.addAll(getTriggers());
 		} else {
 			for (TriggerPolicy<OUT> trigger : getTriggers()) {
@@ -576,10 +600,10 @@ public class WindowedDataStream<OUT> {
 		return cTriggers;
 	}
 
-	private LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
+	protected LinkedList<CloneableTriggerPolicy<OUT>> getDistributedTriggers() {
 		LinkedList<CloneableTriggerPolicy<OUT>> dTriggers = null;
 
-		if (!allCentral) {
+		if (!isCentral) {
 			dTriggers = new LinkedList<CloneableTriggerPolicy<OUT>>();
 			for (TriggerPolicy<OUT> trigger : getTriggers()) {
 				if (!(trigger instanceof TimeTriggerPolicy)) {
@@ -591,10 +615,10 @@ public class WindowedDataStream<OUT> {
 		return dTriggers;
 	}
 
-	private LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
+	protected LinkedList<CloneableEvictionPolicy<OUT>> getDistributedEvicters() {
 		LinkedList<CloneableEvictionPolicy<OUT>> evicters = null;
 
-		if (!allCentral) {
+		if (!isCentral) {
 			evicters = new LinkedList<CloneableEvictionPolicy<OUT>>();
 			for (EvictionPolicy<OUT> evicter : getEvicters()) {
 				evicters.add((CloneableEvictionPolicy<OUT>) evicter);
@@ -604,41 +628,14 @@ public class WindowedDataStream<OUT> {
 		return evicters;
 	}
 
-	private LinkedList<EvictionPolicy<OUT>> getCentralEvicters() {
-		if (allCentral) {
+	protected LinkedList<EvictionPolicy<OUT>> getCentralEvicters() {
+		if (isCentral) {
 			return getEvicters();
 		} else {
 			return null;
 		}
 	}
 
-	private <R> StreamInvokable<OUT, R> getReduceGroupInvokable(GroupReduceFunction<OUT, R> reducer) {
-		StreamInvokable<OUT, R> invokable;
-		if (isGrouped) {
-			invokable = new GroupedWindowInvokable<OUT, R>(clean(reducer), keySelector,
-					getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
-					getCentralEvicters());
-
-		} else {
-			invokable = new WindowGroupReduceInvokable<OUT, R>(clean(reducer), getTriggers(),
-					getEvicters());
-		}
-		return invokable;
-	}
-
-	private StreamInvokable<OUT, OUT> getReduceInvokable(ReduceFunction<OUT> reducer) {
-		StreamInvokable<OUT, OUT> invokable;
-		if (isGrouped) {
-			invokable = new GroupedWindowInvokable<OUT, OUT>(clean(reducer), keySelector,
-					getDistributedTriggers(), getDistributedEvicters(), getCentralTriggers(),
-					getCentralEvicters());
-
-		} else {
-			invokable = new WindowReduceInvokable<OUT>(clean(reducer), getTriggers(), getEvicters());
-		}
-		return invokable;
-	}
-
 	/**
 	 * Gets the output type.
 	 * 
@@ -648,11 +645,30 @@ public class WindowedDataStream<OUT> {
 		return dataStream.getType();
 	}
 
-	public DataStream<OUT> getDataStream() {
+	protected DataStream<OUT> getDataStream() {
 		return dataStream;
 	}
 
 	protected WindowedDataStream<OUT> copy() {
 		return new WindowedDataStream<OUT>(this);
 	}
+
+	protected boolean isGrouped() {
+		return groupByKey != null;
+	}
+
+	public DataStream<StreamWindow<OUT>> getDiscretizedStream() {
+		return discretize(true).getDiscretizedStream();
+	}
+
+	protected static class WindowKey<R> implements KeySelector<StreamWindow<R>, Integer> {
+
+		private static final long serialVersionUID = 1L;
+		
+		@Override
+		public Integer getKey(StreamWindow<R> value) throws Exception {
+			return value.windowID;
+		}
+
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
deleted file mode 100644
index df2edd2..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokable.java
+++ /dev/null
@@ -1,506 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.streamrecord.StreamRecord;
-import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-/**
- * This invokable allows windowing based on {@link TriggerPolicy} and
- * {@link EvictionPolicy} instances including their active and cloneable
- * versions. It is additionally aware of the creation of windows per group.
- * 
- * A {@link KeySelector} is used to specify the key position or key extraction.
- * The {@link ReduceFunction} will be executed on each group separately.
- * Policies might either be centralized or distributed. It is not possible to
- * use central and distributed eviction policies at the same time. A distributed
- * policy have to be a {@link CloneableTriggerPolicy} or
- * {@link CloneableEvictionPolicy} as it will be cloned to have separated
- * instances for each group. At the startup time the distributed policies will
- * be stored as sample, and only clones of them will be used to maintain the
- * groups. Therefore, each group starts with the initial policy states.
- * 
- * While a distributed policy only gets notified with the elements belonging to
- * the respective group, a centralized policy get notified with all arriving
- * elements. When a centralized trigger occurred, all groups get triggered. This
- * is done by submitting the element which caused the trigger as real element to
- * the groups it belongs to and as fake element to all other groups. Within the
- * groups the element might be further processed, causing more triggers,
- * prenotifications of active distributed policies and evictions like usual.
- * 
- * Central policies can be instance of {@link ActiveTriggerPolicy} and also
- * implement the
- * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
- * method. Fake elements created on prenotification will be forwarded to all
- * groups. The {@link ActiveTriggerCallback} is also implemented in a way, that
- * it forwards/distributed calls all groups.
- * 
- * @param <IN>
- *            The type of input elements handled by this operator invokable.
- */
-public class GroupedWindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
-
-	/**
-	 * Auto-generated serial version UID
-	 */
-	private static final long serialVersionUID = -3469545957144404137L;
-
-	private KeySelector<IN, ?> keySelector;
-	private Configuration parameters;
-	private LinkedList<ActiveTriggerPolicy<IN>> activeCentralTriggerPolicies;
-	private LinkedList<TriggerPolicy<IN>> centralTriggerPolicies;
-	private LinkedList<ActiveEvictionPolicy<IN>> activeCentralEvictionPolicies;
-	private LinkedList<EvictionPolicy<IN>> centralEvictionPolicies;
-	private LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies;
-	private LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies;
-	private Map<Object, WindowInvokable<IN, OUT>> windowingGroups;
-	private LinkedList<Thread> activePolicyThreads;
-	private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
-	private LinkedList<WindowInvokable<IN, OUT>> deleteOrderForCentralEviction;
-
-	/**
-	 * This constructor creates an instance of the grouped windowing invokable.
-	 * 
-	 * A {@link KeySelector} is used to specify the key position or key
-	 * extraction. The {@link ReduceFunction} will be executed on each group
-	 * separately. Policies might either be centralized or distributed. It is
-	 * not possible to use central and distributed eviction policies at the same
-	 * time. A distributed policy have to be a {@link CloneableTriggerPolicy} or
-	 * {@link CloneableEvictionPolicy} as it will be cloned to have separated
-	 * instances for each group. At the startup time the distributed policies
-	 * will be stored as sample, and only clones of them will be used to
-	 * maintain the groups. Therefore, each group starts with the initial policy
-	 * states.
-	 * 
-	 * While a distributed policy only gets notified with the elements belonging
-	 * to the respective group, a centralized policy get notified with all
-	 * arriving elements. When a centralized trigger occurred, all groups get
-	 * triggered. This is done by submitting the element which caused the
-	 * trigger as real element to the groups it belongs to and as fake element
-	 * to all other groups. Within the groups the element might be further
-	 * processed, causing more triggers, prenotifications of active distributed
-	 * policies and evictions like usual.
-	 * 
-	 * Central policies can be instance of {@link ActiveTriggerPolicy} and also
-	 * implement the
-	 * {@link ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)}
-	 * method. Fake elements created on prenotification will be forwarded to all
-	 * groups. The {@link ActiveTriggerCallback} is also implemented in a way,
-	 * that it forwards/distributed calls all groups.
-	 * 
-	 * @param userFunction
-	 *            The user defined function.
-	 * @param keySelector
-	 *            A key selector to extract the key for the groups from the
-	 *            input data.
-	 * @param distributedTriggerPolicies
-	 *            Trigger policies to be distributed and maintained individually
-	 *            within each group.
-	 * @param distributedEvictionPolicies
-	 *            Eviction policies to be distributed and maintained
-	 *            individually within each group. Note that there cannot be
-	 *            both, central and distributed eviction policies at the same
-	 *            time.
-	 * @param centralTriggerPolicies
-	 *            Trigger policies which will only exist once at a central
-	 *            place. In case a central policy triggers, it will cause all
-	 *            groups to be emitted. (Remark: Empty groups cannot be emitted.
-	 *            If only one element is contained a group, this element itself
-	 *            is returned as aggregated result.)
-	 * @param centralEvictionPolicies
-	 *            Eviction which will only exist once at a central place. Note
-	 *            that there cannot be both, central and distributed eviction
-	 *            policies at the same time. The central eviction policy will
-	 *            work on an simulated element buffer containing all elements no
-	 *            matter which group they belong to.
-	 */
-	public GroupedWindowInvokable(Function userFunction, KeySelector<IN, ?> keySelector,
-			LinkedList<CloneableTriggerPolicy<IN>> distributedTriggerPolicies,
-			LinkedList<CloneableEvictionPolicy<IN>> distributedEvictionPolicies,
-			LinkedList<TriggerPolicy<IN>> centralTriggerPolicies,
-			LinkedList<EvictionPolicy<IN>> centralEvictionPolicies) {
-
-		super(userFunction);
-
-		this.keySelector = keySelector;
-
-		// handle the triggers
-		if (centralTriggerPolicies != null) {
-			this.centralTriggerPolicies = centralTriggerPolicies;
-			this.activeCentralTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
-
-			for (TriggerPolicy<IN> trigger : centralTriggerPolicies) {
-				if (trigger instanceof ActiveTriggerPolicy) {
-					this.activeCentralTriggerPolicies.add((ActiveTriggerPolicy<IN>) trigger);
-				}
-			}
-		} else {
-			this.centralTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-		}
-
-		if (distributedTriggerPolicies != null) {
-			this.distributedTriggerPolicies = distributedTriggerPolicies;
-		} else {
-			this.distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<IN>>();
-		}
-
-		if (distributedEvictionPolicies != null) {
-			this.distributedEvictionPolicies = distributedEvictionPolicies;
-		} else {
-			this.distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<IN>>();
-		}
-
-		this.activeCentralEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
-
-		if (centralEvictionPolicies != null) {
-			this.centralEvictionPolicies = centralEvictionPolicies;
-
-			for (EvictionPolicy<IN> eviction : centralEvictionPolicies) {
-				if (eviction instanceof ActiveEvictionPolicy) {
-					this.activeCentralEvictionPolicies.add((ActiveEvictionPolicy<IN>) eviction);
-				}
-			}
-		} else {
-			this.centralEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
-		}
-
-		this.windowingGroups = new HashMap<Object, WindowInvokable<IN, OUT>>();
-		this.activePolicyThreads = new LinkedList<Thread>();
-		this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-		this.deleteOrderForCentralEviction = new LinkedList<WindowInvokable<IN, OUT>>();
-
-		// check that not both, central and distributed eviction, is used at the
-		// same time.
-		if (!this.centralEvictionPolicies.isEmpty() && !this.distributedEvictionPolicies.isEmpty()) {
-			throw new UnsupportedOperationException(
-					"You can only use either central or distributed eviction policies but not both at the same time.");
-		}
-
-		// Check that there is at least one trigger and one eviction policy
-		if (this.centralEvictionPolicies.isEmpty() && this.distributedEvictionPolicies.isEmpty()) {
-			throw new UnsupportedOperationException(
-					"You have to define at least one eviction policy");
-		}
-		if (this.centralTriggerPolicies.isEmpty() && this.distributedTriggerPolicies.isEmpty()) {
-			throw new UnsupportedOperationException(
-					"You have to define at least one trigger policy");
-		}
-
-	}
-
-	@Override
-	public void invoke() throws Exception {
-		// Prevent empty data streams
-		if (readNext() == null) {
-			throw new RuntimeException("DataStream must not be empty");
-		}
-
-		// Continuously run
-		while (nextRecord != null) {
-			WindowInvokable<IN, OUT> groupInvokable = windowingGroups.get(keySelector
-					.getKey(nextRecord.getObject()));
-			if (groupInvokable == null) {
-				groupInvokable = makeNewGroup(nextRecord);
-			}
-
-			// Run the precalls for central active triggers
-			for (ActiveTriggerPolicy<IN> trigger : activeCentralTriggerPolicies) {
-				Object[] result = trigger.preNotifyTrigger(nextRecord.getObject());
-				for (Object in : result) {
-
-					// If central eviction is used, handle it here
-					if (!activeCentralEvictionPolicies.isEmpty()) {
-						evictElements(centralActiveEviction(in));
-					}
-
-					// process in groups
-					for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
-						group.processFakeElement(in, trigger);
-						checkForEmptyGroupBuffer(group);
-					}
-				}
-			}
-
-			// Process non-active central triggers
-			for (TriggerPolicy<IN> triggerPolicy : centralTriggerPolicies) {
-				if (triggerPolicy.notifyTrigger(nextRecord.getObject())) {
-					currentTriggerPolicies.add(triggerPolicy);
-				}
-			}
-
-			if (currentTriggerPolicies.isEmpty()) {
-
-				// only add the element to its group
-				groupInvokable.processRealElement(nextRecord.getObject());
-				checkForEmptyGroupBuffer(groupInvokable);
-
-				// If central eviction is used, handle it here
-				if (!centralEvictionPolicies.isEmpty()) {
-					evictElements(centralEviction(nextRecord.getObject(), false));
-					deleteOrderForCentralEviction.add(groupInvokable);
-				}
-
-			} else {
-
-				// call user function for all groups
-				for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
-					if (group == groupInvokable) {
-						// process real with initialized policies
-						group.processRealElement(nextRecord.getObject(), currentTriggerPolicies);
-					} else {
-						// process like a fake but also initialized with
-						// policies
-						group.externalTriggerFakeElement(nextRecord.getObject(),
-								currentTriggerPolicies);
-					}
-
-					// remove group in case it has an empty buffer
-					// checkForEmptyGroupBuffer(group);
-				}
-
-				// If central eviction is used, handle it here
-				if (!centralEvictionPolicies.isEmpty()) {
-					evictElements(centralEviction(nextRecord.getObject(), true));
-					deleteOrderForCentralEviction.add(groupInvokable);
-				}
-			}
-
-			// clear current trigger list
-			currentTriggerPolicies.clear();
-
-			// read next record
-			readNext();
-		}
-
-		// Stop all remaining threads from policies
-		for (Thread t : activePolicyThreads) {
-			t.interrupt();
-		}
-
-		// finally trigger the buffer.
-		for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
-			group.emitFinalWindow(centralTriggerPolicies);
-		}
-
-	}
-
-	/**
-	 * This method creates a new group. The method gets called in case an
-	 * element arrives which has a key which was not seen before. The method
-	 * created a nested {@link WindowInvokable} and therefore created clones of
-	 * all distributed trigger and eviction policies.
-	 * 
-	 * @param element
-	 *            The element which leads to the generation of a new group
-	 *            (previously unseen key)
-	 * @throws Exception
-	 *             In case the {@link KeySelector} throws an exception in
-	 *             {@link KeySelector#getKey(Object)}, the exception is not
-	 *             catched by this method.
-	 */
-	@SuppressWarnings("unchecked")
-	private WindowInvokable<IN, OUT> makeNewGroup(StreamRecord<IN> element) throws Exception {
-		// clone the policies
-		LinkedList<TriggerPolicy<IN>> clonedDistributedTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-		LinkedList<EvictionPolicy<IN>> clonedDistributedEvictionPolicies = new LinkedList<EvictionPolicy<IN>>();
-		for (CloneableTriggerPolicy<IN> trigger : this.distributedTriggerPolicies) {
-			clonedDistributedTriggerPolicies.add(trigger.clone());
-		}
-		for (CloneableEvictionPolicy<IN> eviction : this.distributedEvictionPolicies) {
-			clonedDistributedEvictionPolicies.add(eviction.clone());
-		}
-
-		WindowInvokable<IN, OUT> groupInvokable;
-		if (userFunction instanceof ReduceFunction) {
-			groupInvokable = (WindowInvokable<IN, OUT>) new WindowReduceInvokable<IN>(
-					(ReduceFunction<IN>) userFunction, clonedDistributedTriggerPolicies,
-					clonedDistributedEvictionPolicies);
-		} else {
-			groupInvokable = new WindowGroupReduceInvokable<IN, OUT>(
-					(GroupReduceFunction<IN, OUT>) userFunction, clonedDistributedTriggerPolicies,
-					clonedDistributedEvictionPolicies);
-		}
-
-		groupInvokable.setup(taskContext, executionConfig);
-		groupInvokable.open(this.parameters);
-		windowingGroups.put(keySelector.getKey(element.getObject()), groupInvokable);
-
-		return groupInvokable;
-	}
-
-	@Override
-	public void open(Configuration parameters) throws Exception {
-		super.open(parameters);
-		this.parameters = parameters;
-		for (ActiveTriggerPolicy<IN> tp : activeCentralTriggerPolicies) {
-			Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
-			if (target != null) {
-				Thread thread = new Thread(target);
-				activePolicyThreads.add(thread);
-				thread.start();
-			}
-		}
-	};
-
-	/**
-	 * This method is used to notify central eviction policies with a real
-	 * element.
-	 * 
-	 * @param input
-	 *            the real element to notify the eviction policy.
-	 * @param triggered
-	 *            whether a central trigger occurred or not.
-	 * @return The number of elements to be deleted from the buffer.
-	 */
-	private int centralEviction(IN input, boolean triggered) {
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (EvictionPolicy<IN> evictionPolicy : centralEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEviction(input, triggered,
-					deleteOrderForCentralEviction.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
-		return currentMaxEviction;
-	}
-
-	/**
-	 * This method is used to notify active central eviction policies with a
-	 * fake element.
-	 * 
-	 * @param input
-	 *            the fake element to notify the active central eviction
-	 *            policies.
-	 * @return The number of elements to be deleted from the buffer.
-	 */
-	private int centralActiveEviction(Object input) {
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (ActiveEvictionPolicy<IN> evictionPolicy : activeCentralEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input,
-					deleteOrderForCentralEviction.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
-		return currentMaxEviction;
-	}
-
-	/**
-	 * This method is used in central eviction to delete a given number of
-	 * elements from the buffer.
-	 * 
-	 * @param numToEvict
-	 *            number of elements to delete from the virtual central element
-	 *            buffer.
-	 */
-	private void evictElements(int numToEvict) {
-		HashSet<WindowInvokable<IN, OUT>> usedGroups = new HashSet<WindowInvokable<IN, OUT>>();
-		for (; numToEvict > 0; numToEvict--) {
-			WindowInvokable<IN, OUT> currentGroup = deleteOrderForCentralEviction.getFirst();
-			// Do the eviction
-			currentGroup.evictFirst();
-			// Remember groups which possibly have an empty buffer after the
-			// eviction
-			usedGroups.add(currentGroup);
-			try {
-				deleteOrderForCentralEviction.removeFirst();
-			} catch (NoSuchElementException e) {
-				// when buffer is empty, ignore exception and stop deleting
-				break;
-			}
-
-		}
-
-		// Remove groups with empty buffer
-		for (WindowInvokable<IN, OUT> group : usedGroups) {
-			checkForEmptyGroupBuffer(group);
-		}
-	}
-
-	/**
-	 * Checks if the element buffer of a given windowing group is empty. If so,
-	 * the group will be deleted.
-	 * 
-	 * @param group
-	 *            The windowing group to be checked and and removed in case its
-	 *            buffer is empty.
-	 */
-	private void checkForEmptyGroupBuffer(WindowInvokable<IN, OUT> group) {
-		if (group.isBufferEmpty()) {
-			windowingGroups.remove(group);
-		}
-	}
-
-	/**
-	 * This callback class allows to handle the callbacks done by threads
-	 * defined in active trigger policies
-	 * 
-	 * @see ActiveTriggerPolicy#createActiveTriggerRunnable(ActiveTriggerCallback)
-	 */
-	private class WindowingCallback implements ActiveTriggerCallback {
-		private ActiveTriggerPolicy<IN> policy;
-
-		public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
-			this.policy = policy;
-		}
-
-		@Override
-		public void sendFakeElement(Object datapoint) {
-
-			// If central eviction is used, handle it here
-			if (!centralEvictionPolicies.isEmpty()) {
-				evictElements(centralActiveEviction(datapoint));
-			}
-
-			// handle element in groups
-			for (WindowInvokable<IN, OUT> group : windowingGroups.values()) {
-				group.processFakeElement(datapoint, policy);
-				checkForEmptyGroupBuffer(group);
-			}
-		}
-
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
deleted file mode 100644
index b3fdfe8..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowGroupReduceInvokable.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.LinkedList;
-
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public class WindowGroupReduceInvokable<IN, OUT> extends WindowInvokable<IN, OUT> {
-
-	private static final long serialVersionUID = 1L;
-	GroupReduceFunction<IN, OUT> reducer;
-
-	public WindowGroupReduceInvokable(GroupReduceFunction<IN, OUT> userFunction,
-			LinkedList<TriggerPolicy<IN>> triggerPolicies,
-			LinkedList<EvictionPolicy<IN>> evictionPolicies) {
-		super(userFunction, triggerPolicies, evictionPolicies);
-		this.reducer = userFunction;
-	}
-
-	@Override
-	protected void callUserFunction() throws Exception {
-		reducer.reduce(copyBuffer(), collector);
-	}
-
-	public LinkedList<IN> copyBuffer() {
-		LinkedList<IN> copy = new LinkedList<IN>();
-		for (IN element : buffer) {
-			copy.add(copy(element));
-		}
-		return copy;
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
deleted file mode 100644
index ea891c9..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokable.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.invokable.StreamInvokable;
-import org.apache.flink.streaming.api.windowing.policy.ActiveEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerCallback;
-import org.apache.flink.streaming.api.windowing.policy.ActiveTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public abstract class WindowInvokable<IN, OUT> extends StreamInvokable<IN, OUT> {
-
-	/**
-	 * Auto-generated serial version UID
-	 */
-	private static final long serialVersionUID = -8038984294071650730L;
-
-	private LinkedList<TriggerPolicy<IN>> triggerPolicies;
-	private LinkedList<EvictionPolicy<IN>> evictionPolicies;
-	private LinkedList<ActiveTriggerPolicy<IN>> activeTriggerPolicies;
-	private LinkedList<ActiveEvictionPolicy<IN>> activeEvictionPolicies;
-	private LinkedList<Thread> activePolicyTreads;
-	protected LinkedList<IN> buffer;
-	private LinkedList<TriggerPolicy<IN>> currentTriggerPolicies;
-
-	/**
-	 * This constructor created a windowing invokable using trigger and eviction
-	 * policies.
-	 * 
-	 * @param userFunction
-	 *            The user defined {@link ReduceFunction}
-	 * @param triggerPolicies
-	 *            A list of {@link TriggerPolicy}s and/or
-	 *            {@link ActiveTriggerPolicy}s
-	 * @param evictionPolicies
-	 *            A list of {@link EvictionPolicy}s and/or
-	 *            {@link ActiveEvictionPolicy}s
-	 */
-	public WindowInvokable(Function userFunction, LinkedList<TriggerPolicy<IN>> triggerPolicies,
-			LinkedList<EvictionPolicy<IN>> evictionPolicies) {
-		super(userFunction);
-
-		this.triggerPolicies = triggerPolicies;
-		this.evictionPolicies = evictionPolicies;
-
-		activeTriggerPolicies = new LinkedList<ActiveTriggerPolicy<IN>>();
-		for (TriggerPolicy<IN> tp : triggerPolicies) {
-			if (tp instanceof ActiveTriggerPolicy) {
-				activeTriggerPolicies.add((ActiveTriggerPolicy<IN>) tp);
-			}
-		}
-
-		activeEvictionPolicies = new LinkedList<ActiveEvictionPolicy<IN>>();
-		for (EvictionPolicy<IN> ep : evictionPolicies) {
-			if (ep instanceof ActiveEvictionPolicy) {
-				activeEvictionPolicies.add((ActiveEvictionPolicy<IN>) ep);
-			}
-		}
-
-		this.activePolicyTreads = new LinkedList<Thread>();
-		this.buffer = new LinkedList<IN>();
-		this.currentTriggerPolicies = new LinkedList<TriggerPolicy<IN>>();
-	}
-
-	@Override
-	public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
-		super.open(parameters);
-		for (ActiveTriggerPolicy<IN> tp : activeTriggerPolicies) {
-			Runnable target = tp.createActiveTriggerRunnable(new WindowingCallback(tp));
-			if (target != null) {
-				Thread thread = new Thread(target);
-				activePolicyTreads.add(thread);
-				thread.start();
-			}
-		}
-	};
-
-	/**
-	 * This class allows the active trigger threads to call back and push fake
-	 * elements at any time.
-	 */
-	private class WindowingCallback implements ActiveTriggerCallback {
-		private ActiveTriggerPolicy<IN> policy;
-
-		public WindowingCallback(ActiveTriggerPolicy<IN> policy) {
-			this.policy = policy;
-		}
-
-		@Override
-		public void sendFakeElement(Object datapoint) {
-			processFakeElement(datapoint, this.policy);
-		}
-
-	}
-
-	@Override
-	public void invoke() throws Exception {
-
-		// Prevent empty data streams
-		if (readNext() == null) {
-			throw new RuntimeException("DataStream must not be empty");
-		}
-
-		// Continuously run
-		while (nextRecord != null) {
-			processRealElement(nextRecord.getObject());
-
-			// Load next StreamRecord
-			readNext();
-		}
-
-		// Stop all remaining threads from policies
-		for (Thread t : activePolicyTreads) {
-			t.interrupt();
-		}
-
-		// finally trigger the buffer.
-		emitFinalWindow(null);
-
-	}
-
-	/**
-	 * This method gets called in case of an grouped windowing in case central
-	 * trigger occurred and the arriving element causing the trigger is not part
-	 * of this group.
-	 * 
-	 * Remark: This is NOT the same as
-	 * {@link WindowInvokable#processFakeElement(Object, TriggerPolicy)}! Here
-	 * the eviction using active policies takes place after the call to the UDF.
-	 * Usually it is done before when fake elements get submitted. This special
-	 * behaviour is needed to allow the {@link GroupedWindowInvokable} to send
-	 * central triggers to all groups, even if the current element does not
-	 * belong to the group.
-	 * 
-	 * @param input
-	 *            a fake input element
-	 * @param policies
-	 *            the list of policies which caused the call with this fake
-	 *            element
-	 */
-	protected synchronized void externalTriggerFakeElement(IN input,
-			List<TriggerPolicy<IN>> policies) {
-
-		// Set the current triggers
-		currentTriggerPolicies.addAll(policies);
-
-		// emit
-		callUserFunctionAndLogException();
-
-		// clear the flag collection
-		currentTriggerPolicies.clear();
-
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
-
-		for (int i = 0; i < currentMaxEviction; i++) {
-			try {
-				buffer.removeFirst();
-			} catch (NoSuchElementException e) {
-				// In case no more elements are in the buffer:
-				// Prevent failure and stop deleting.
-				break;
-			}
-		}
-	}
-
-	/**
-	 * This method processed an arrived fake element The method is synchronized
-	 * to ensure that it cannot interleave with
-	 * {@link WindowInvokable#processRealElement(Object)}
-	 * 
-	 * @param input
-	 *            a fake input element
-	 * @param currentPolicy
-	 *            the policy which produced this fake element
-	 */
-	protected synchronized void processFakeElement(Object input, TriggerPolicy<IN> currentPolicy) {
-
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-		for (ActiveEvictionPolicy<IN> evictionPolicy : activeEvictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEvictionWithFakeElement(input, buffer.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
-
-		for (int i = 0; i < currentMaxEviction; i++) {
-			try {
-				buffer.removeFirst();
-			} catch (NoSuchElementException e) {
-				// In case no more elements are in the buffer:
-				// Prevent failure and stop deleting.
-				break;
-			}
-		}
-
-		// Set the current trigger
-		currentTriggerPolicies.add(currentPolicy);
-
-		// emit
-		callUserFunctionAndLogException();
-
-		// clear the flag collection
-		currentTriggerPolicies.clear();
-	}
-
-	/**
-	 * This method processed an arrived real element The method is synchronized
-	 * to ensure that it cannot interleave with
-	 * {@link WindowInvokable#processFakeElement(Object)}.
-	 * 
-	 * @param input
-	 *            a real input element
-	 * @param triggerPolicies
-	 *            Allows to set trigger policies which are maintained
-	 *            externally. This is the case for central policies in
-	 *            {@link GroupedWindowInvokable}.
-	 */
-	protected synchronized void processRealElement(IN input, List<TriggerPolicy<IN>> triggerPolicies) {
-		this.currentTriggerPolicies.addAll(triggerPolicies);
-		processRealElement(input);
-	}
-
-	/**
-	 * This method processed an arrived real element The method is synchronized
-	 * to ensure that it cannot interleave with
-	 * {@link WindowInvokable#processFakeElement(Object)}
-	 * 
-	 * @param input
-	 *            a real input element
-	 */
-	protected synchronized void processRealElement(IN input) {
-
-		// Run the precalls to detect missed windows
-		for (ActiveTriggerPolicy<IN> trigger : activeTriggerPolicies) {
-			// Remark: In case multiple active triggers are present the ordering
-			// of the different fake elements returned by this triggers becomes
-			// a problem. This might lead to unexpected results...
-			// Should we limit the number of active triggers to 0 or 1?
-			Object[] result = trigger.preNotifyTrigger(input);
-			for (Object in : result) {
-				processFakeElement(in, trigger);
-			}
-		}
-
-		// Remember if a trigger occurred
-		boolean isTriggered = false;
-
-		// Process the triggers
-		for (TriggerPolicy<IN> triggerPolicy : triggerPolicies) {
-			if (triggerPolicy.notifyTrigger(input)) {
-				currentTriggerPolicies.add(triggerPolicy);
-			}
-		}
-
-		// call user function
-		if (!currentTriggerPolicies.isEmpty()) {
-			// emit
-			callUserFunctionAndLogException();
-
-			// clear the flag collection
-			currentTriggerPolicies.clear();
-
-			// remember trigger
-			isTriggered = true;
-		}
-
-		// Process the evictions and take care of double evictions
-		// In case there are multiple eviction policies present,
-		// only the one with the highest return value is recognized.
-		int currentMaxEviction = 0;
-
-		for (EvictionPolicy<IN> evictionPolicy : evictionPolicies) {
-			// use temporary variable to prevent multiple calls to
-			// notifyEviction
-			int tmp = evictionPolicy.notifyEviction(input, isTriggered, buffer.size());
-			if (tmp > currentMaxEviction) {
-				currentMaxEviction = tmp;
-			}
-		}
-
-		for (int i = 0; i < currentMaxEviction; i++) {
-			try {
-				buffer.removeFirst();
-			} catch (NoSuchElementException e) {
-				// In case no more elements are in the buffer:
-				// Prevent failure and stop deleting.
-				break;
-			}
-		}
-
-		// Add the current element to the buffer
-		buffer.add(input);
-
-	}
-
-	/**
-	 * This method removes the first element from the element buffer. It is used
-	 * to provide central evictions in {@link GroupedWindowInvokable}
-	 */
-	protected synchronized void evictFirst() {
-		try {
-			buffer.removeFirst();
-		} catch (NoSuchElementException e) {
-			// ignore exception
-		}
-	}
-
-	/**
-	 * This method returns whether the element buffer is empty or not. It is
-	 * used to figure out if a group can be deleted or not when
-	 * {@link GroupedWindowInvokable} is used.
-	 * 
-	 * @return true in case the buffer is empty otherwise false.
-	 */
-	protected boolean isBufferEmpty() {
-		return buffer.isEmpty();
-	}
-
-	/**
-	 * This method does the final reduce at the end of the stream and emits the
-	 * result.
-	 * 
-	 * @param centralTriggerPolicies
-	 *            Allows to set trigger policies which are maintained
-	 *            externally. This is the case for central policies in
-	 *            {@link GroupedWindowInvokable}.
-	 */
-	protected void emitFinalWindow(List<TriggerPolicy<IN>> centralTriggerPolicies) {
-		if (!buffer.isEmpty()) {
-			currentTriggerPolicies.clear();
-
-			if (centralTriggerPolicies != null) {
-				currentTriggerPolicies.addAll(centralTriggerPolicies);
-			}
-
-			for (TriggerPolicy<IN> policy : triggerPolicies) {
-				currentTriggerPolicies.add(policy);
-			}
-
-			callUserFunctionAndLogException();
-		}
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
deleted file mode 100644
index ed246c8..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/WindowReduceInvokable.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import java.util.Iterator;
-import java.util.LinkedList;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-
-public class WindowReduceInvokable<IN> extends WindowInvokable<IN, IN> {
-
-	private static final long serialVersionUID = 1L;
-
-	ReduceFunction<IN> reducer;
-
-	public WindowReduceInvokable(ReduceFunction<IN> userFunction,
-			LinkedList<TriggerPolicy<IN>> triggerPolicies,
-			LinkedList<EvictionPolicy<IN>> evictionPolicies) {
-		super(userFunction, triggerPolicies, evictionPolicies);
-		this.reducer = userFunction;
-	}
-
-	@Override
-	protected void callUserFunction() throws Exception {
-		Iterator<IN> reducedIterator = buffer.iterator();
-		IN reduced = null;
-
-		while (reducedIterator.hasNext() && reduced == null) {
-			reduced = reducedIterator.next();
-		}
-
-		while (reducedIterator.hasNext()) {
-			IN next = reducedIterator.next();
-			if (next != null) {
-				reduced = reducer.reduce(copy(reduced), copy(next));
-			}
-		}
-		if (reduced != null) {
-			collector.collect(reduced);
-		}
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
index 738c78f..1766b0b 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowMerger.java
@@ -22,11 +22,11 @@ import java.util.Map;
 
 import org.apache.flink.streaming.api.invokable.ChainableInvokable;
 
-public class WindowCombiner<T> extends ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
+public class WindowMerger<T> extends ChainableInvokable<StreamWindow<T>, StreamWindow<T>> {
 
 	private Map<Integer, StreamWindow<T>> windows;
 
-	public WindowCombiner() {
+	public WindowMerger() {
 		super(null);
 		this.windows = new HashMap<Integer, StreamWindow<T>>();
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
index 132b495..1937b3f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableEvictionPolicy.java
@@ -17,7 +17,6 @@
 
 package org.apache.flink.streaming.api.windowing.policy;
 
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
 
 /**
  * When used in {@link GroupedWindowInvokable}, eviction policies must

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
index f5772a1..6a04461 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/policy/CloneableTriggerPolicy.java
@@ -17,7 +17,6 @@
 
 package org.apache.flink.streaming.api.windowing.policy;
 
-import org.apache.flink.streaming.api.invokable.operator.GroupedWindowInvokable;
 
 /**
  * When used in {@link GroupedWindowInvokable}, trigger policies can provide


[16/20] flink git commit: [streaming] WindowMapFunction added + Streaming package structure cleanup

Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowSerializer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowSerializer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowSerializer.java
new file mode 100755
index 0000000..5b40e53
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowSerializer.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing;
+
+import java.io.IOException;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+public final class StreamWindowSerializer<T> extends TypeSerializer<StreamWindow<T>> {
+
+	private static final long serialVersionUID = 1L;
+
+	private final TypeSerializer<T> typeSerializer;
+	TypeSerializer<Integer> intSerializer;
+
+	public StreamWindowSerializer(TypeInformation<T> typeInfo, ExecutionConfig conf) {
+		this.typeSerializer = typeInfo.createSerializer(conf);
+		this.intSerializer = BasicTypeInfo.INT_TYPE_INFO.createSerializer(conf);
+	}
+
+	public TypeSerializer<T> getObjectSerializer() {
+		return typeSerializer;
+	}
+
+	@Override
+	public boolean isImmutableType() {
+		return false;
+	}
+
+	@Override
+	public StreamWindow<T> createInstance() {
+		return new StreamWindow<T>(0, 0);
+	}
+
+	@Override
+	public StreamWindow<T> copy(StreamWindow<T> from) {
+		return new StreamWindow<T>(from, typeSerializer);
+	}
+
+	@Override
+	public StreamWindow<T> copy(StreamWindow<T> from, StreamWindow<T> reuse) {
+		reuse.clear();
+		reuse.windowID = from.windowID;
+		reuse.numberOfParts = from.numberOfParts;
+
+		for (T element : from) {
+			reuse.add(typeSerializer.copy(element));
+		}
+		return reuse;
+	}
+
+	@Override
+	public int getLength() {
+		return -1;
+	}
+
+	@Override
+	public void serialize(StreamWindow<T> value, DataOutputView target) throws IOException {
+
+		intSerializer.serialize(value.windowID, target);
+		intSerializer.serialize(value.numberOfParts, target);
+		intSerializer.serialize(value.size(), target);
+
+		for (T element : value) {
+			typeSerializer.serialize(element, target);
+		}
+	}
+
+	@Override
+	public StreamWindow<T> deserialize(DataInputView source) throws IOException {
+		StreamWindow<T> window = createInstance();
+
+		window.windowID = intSerializer.deserialize(source);
+		window.numberOfParts = intSerializer.deserialize(source);
+
+		int size = intSerializer.deserialize(source);
+
+		for (int i = 0; i < size; i++) {
+			window.add(typeSerializer.deserialize(source));
+		}
+
+		return window;
+	}
+
+	@Override
+	public StreamWindow<T> deserialize(StreamWindow<T> reuse, DataInputView source)
+			throws IOException {
+
+		StreamWindow<T> window = reuse;
+		window.clear();
+
+		window.windowID = source.readInt();
+		window.numberOfParts = source.readInt();
+
+		int size = source.readInt();
+
+		for (int i = 0; i < size; i++) {
+			window.add(typeSerializer.deserialize(source));
+		}
+
+		return window;
+	}
+
+	@Override
+	public void copy(DataInputView source, DataOutputView target) throws IOException {
+		// Needs to be implemented
+	}
+
+	@Override
+	public TypeSerializer<StreamWindow<T>> duplicate() {
+		return this;
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowTypeInfo.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowTypeInfo.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowTypeInfo.java
new file mode 100644
index 0000000..3bcc253
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/StreamWindowTypeInfo.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+
+public class StreamWindowTypeInfo<T> extends TypeInformation<StreamWindow<T>> {
+
+	private static final long serialVersionUID = 1L;
+	TypeInformation<T> innerType;
+
+	public StreamWindowTypeInfo(TypeInformation<T> innerType) {
+		this.innerType = innerType;
+	}
+
+	public TypeInformation<T> getInnerType() {
+		return innerType;
+	}
+
+	@Override
+	public boolean isBasicType() {
+		return innerType.isBasicType();
+	}
+
+	@Override
+	public boolean isTupleType() {
+		return innerType.isTupleType();
+	}
+
+	@Override
+	public int getArity() {
+		return innerType.getArity();
+	}
+
+	@Override
+	public Class<StreamWindow<T>> getTypeClass() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	@Override
+	public boolean isKeyType() {
+		return innerType.isKeyType();
+	}
+
+	@Override
+	public TypeSerializer<StreamWindow<T>> createSerializer(ExecutionConfig conf) {
+		return new StreamWindowSerializer<T>(innerType, conf);
+	}
+
+	@Override
+	public int getTotalFields() {
+		return innerType.getTotalFields();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java
new file mode 100644
index 0000000..9d5676c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/BasicWindowBuffer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+import java.util.LinkedList;
+import java.util.NoSuchElementException;
+
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.util.Collector;
+
+public class BasicWindowBuffer<T> implements WindowBuffer<T> {
+
+	private static final long serialVersionUID = 1L;
+	protected LinkedList<T> buffer;
+
+	public BasicWindowBuffer() {
+		this.buffer = new LinkedList<T>();
+	}
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
+		if (!buffer.isEmpty()) {
+			StreamWindow<T> currentWindow = new StreamWindow<T>();
+			currentWindow.addAll(buffer);
+			collector.collect(currentWindow);
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	public void store(T element) throws Exception {
+		buffer.add(element);
+	}
+
+	public void evict(int n) {
+		for (int i = 0; i < n; i++) {
+			try {
+				buffer.removeFirst();
+			} catch (NoSuchElementException e) {
+				// In case no more elements are in the buffer:
+				// Prevent failure and stop deleting.
+				break;
+			}
+		}
+	}
+
+	public int size() {
+		return buffer.size();
+	}
+
+	@Override
+	public BasicWindowBuffer<T> clone() {
+		return new BasicWindowBuffer<T>();
+	}
+
+	@Override
+	public String toString() {
+		return buffer.toString();
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java
new file mode 100644
index 0000000..a01f97c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/CompletePreAggregator.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUStreamRecord<?>WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+public interface CompletePreAggregator {
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java
new file mode 100644
index 0000000..eef8bed
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingGroupedPreReducer.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.util.Collector;
+
+public class TumblingGroupedPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
+
+	private static final long serialVersionUID = 1L;
+
+	private ReduceFunction<T> reducer;
+	private KeySelector<T, ?> keySelector;
+
+	private Map<Object, T> reducedValues;
+
+	private int numOfElements = 0;
+	private TypeSerializer<T> serializer;
+
+	public TumblingGroupedPreReducer(ReduceFunction<T> reducer, KeySelector<T, ?> keySelector,
+			TypeSerializer<T> serializer) {
+		this.reducer = reducer;
+		this.serializer = serializer;
+		this.keySelector = keySelector;
+		this.reducedValues = new HashMap<Object, T>();
+	}
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
+
+		if (!reducedValues.isEmpty()) {
+			StreamWindow<T> currentWindow = new StreamWindow<T>();
+			currentWindow.addAll(reducedValues.values());
+			collector.collect(currentWindow);
+			reducedValues.clear();
+			numOfElements = 0;
+			return true;
+		} else {
+			return false;
+		}
+
+	}
+
+	public void store(T element) throws Exception {
+		Object key = keySelector.getKey(element);
+
+		T reduced = reducedValues.get(key);
+
+		if (reduced == null) {
+			reduced = element;
+		} else {
+			reduced = reducer.reduce(serializer.copy(reduced), element);
+		}
+
+		reducedValues.put(key, reduced);
+		numOfElements++;
+	}
+
+	public void evict(int n) {
+	}
+
+	public int size() {
+		return numOfElements;
+	}
+
+	@Override
+	public TumblingGroupedPreReducer<T> clone() {
+		return new TumblingGroupedPreReducer<T>(reducer, keySelector, serializer);
+	}
+
+	@Override
+	public String toString() {
+		return reducedValues.toString();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java
new file mode 100644
index 0000000..b755e73
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/TumblingPreReducer.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.util.Collector;
+
+public class TumblingPreReducer<T> implements WindowBuffer<T>, CompletePreAggregator {
+
+	private static final long serialVersionUID = 1L;
+
+	private ReduceFunction<T> reducer;
+
+	private T reduced;
+	private int numOfElements = 0;
+	private TypeSerializer<T> serializer;
+
+	public TumblingPreReducer(ReduceFunction<T> reducer, TypeSerializer<T> serializer) {
+		this.reducer = reducer;
+		this.serializer = serializer;
+	}
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector) {
+		if (reduced != null) {
+			StreamWindow<T> currentWindow = new StreamWindow<T>();
+			currentWindow.add(reduced);
+			collector.collect(currentWindow);
+			reduced = null;
+			numOfElements = 0;
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	public void store(T element) throws Exception {
+		if (reduced == null) {
+			reduced = element;
+		} else {
+			reduced = reducer.reduce(serializer.copy(reduced), element);
+		}
+		numOfElements++;
+	}
+
+	public void evict(int n) {
+	}
+
+	public int size() {
+		return numOfElements;
+	}
+
+	@Override
+	public TumblingPreReducer<T> clone() {
+		return new TumblingPreReducer<T>(reducer, serializer);
+	}
+
+	@Override
+	public String toString() {
+		return reduced.toString();
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
new file mode 100644
index 0000000..59abcd6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/windowing/windowbuffer/WindowBuffer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.windowing.windowbuffer;
+
+import java.io.Serializable;
+
+import org.apache.flink.streaming.api.windowing.StreamWindow;
+import org.apache.flink.util.Collector;
+
+public interface WindowBuffer<T> extends Serializable {
+
+	public void store(T element) throws Exception;
+
+	public void evict(int n);
+
+	public boolean emitWindow(Collector<StreamWindow<T>> collector);
+
+	public int size();
+	
+	public WindowBuffer<T> clone();
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
deleted file mode 100644
index a91bd0c..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoFlatMapTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class CoFlatMapTest implements Serializable {
-	private static final long serialVersionUID = 1L;
-
-	private final static class MyCoFlatMap implements CoFlatMapFunction<String, Integer, String> {
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public void flatMap1(String value, Collector<String> coll) {
-			for (int i = 0; i < value.length(); i++) {
-				coll.collect(value.substring(i, i + 1));
-			}
-		}
-
-		@Override
-		public void flatMap2(Integer value, Collector<String> coll) {
-			coll.collect(value.toString());
-		}
-	}
-
-	@Test
-	public void coFlatMapTest() {
-		CoFlatMapInvokable<String, Integer, String> invokable = new CoFlatMapInvokable<String, Integer, String>(
-				new MyCoFlatMap());
-
-		List<String> expectedList = Arrays.asList("a", "b", "c", "1", "d", "e", "f", "2", "g", "h",
-				"e", "3", "4", "5");
-		List<String> actualList = MockCoContext.createAndExecute(invokable,
-				Arrays.asList("abc", "def", "ghe"), Arrays.asList(1, 2, 3, 4, 5));
-
-		assertEquals(expectedList, actualList);
-	}
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void multipleInputTest() {
-		LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
-		DataStream<Integer> ds1 = env.fromElements(1, 3, 5);
-		DataStream<Integer> ds2 = env.fromElements(2, 4).merge(ds1);
-		
-		try {
-			ds1.forward().merge(ds2);
-			fail();
-		} catch (RuntimeException e) {
-			// expected
-		}
-		
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
deleted file mode 100644
index 273bbae..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoGroupedReduceTest.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedReduceInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoGroupedReduceTest {
-
-	private final static class MyCoReduceFunction implements
-			CoReduceFunction<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> {
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public Tuple3<String, String, String> reduce1(Tuple3<String, String, String> value1,
-				Tuple3<String, String, String> value2) {
-			return new Tuple3<String, String, String>(value1.f0, value1.f1 + value2.f1, value1.f2);
-		}
-
-		@Override
-		public Tuple2<Integer, Integer> reduce2(Tuple2<Integer, Integer> value1,
-				Tuple2<Integer, Integer> value2) {
-			return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
-		}
-
-		@Override
-		public String map1(Tuple3<String, String, String> value) {
-			return value.f1;
-		}
-
-		@Override
-		public String map2(Tuple2<Integer, Integer> value) {
-			return value.f1.toString();
-		}
-	}
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void coGroupedReduceTest() {
-		Tuple3<String, String, String> word1 = new Tuple3<String, String, String>("a", "word1", "b");
-		Tuple3<String, String, String> word2 = new Tuple3<String, String, String>("b", "word2", "a");
-		Tuple3<String, String, String> word3 = new Tuple3<String, String, String>("a", "word3", "a");
-		Tuple2<Integer, Integer> int1 = new Tuple2<Integer, Integer>(2, 1);
-		Tuple2<Integer, Integer> int2 = new Tuple2<Integer, Integer>(1, 2);
-		Tuple2<Integer, Integer> int3 = new Tuple2<Integer, Integer>(0, 3);
-		Tuple2<Integer, Integer> int4 = new Tuple2<Integer, Integer>(2, 4);
-		Tuple2<Integer, Integer> int5 = new Tuple2<Integer, Integer>(1, 5);
-
-		KeySelector<Tuple3<String, String, String>, ?> keySelector0 = new KeySelector<Tuple3<String, String, String>, String>() {
-
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public String getKey(Tuple3<String, String, String> value) throws Exception {
-				return value.f0;
-			}
-		};
-
-		KeySelector<Tuple2<Integer, Integer>, ?> keySelector1 = new KeySelector<Tuple2<Integer, Integer>, Integer>() {
-
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
-				return value.f0;
-			}
-		};
-
-		KeySelector<Tuple3<String, String, String>, ?> keySelector2 = new KeySelector<Tuple3<String, String, String>, String>() {
-
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public String getKey(Tuple3<String, String, String> value) throws Exception {
-				return value.f2;
-			}
-		};
-
-		CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
-				new MyCoReduceFunction(), keySelector0, keySelector1);
-
-		List<String> expected = Arrays.asList("word1", "1", "word2", "2", "word1word3", "3", "5",
-				"7");
-
-		List<String> actualList = MockCoContext.createAndExecute(invokable,
-				Arrays.asList(word1, word2, word3), Arrays.asList(int1, int2, int3, int4, int5));
-
-		assertEquals(expected, actualList);
-
-		invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
-				new MyCoReduceFunction(), keySelector2, keySelector1);
-
-		expected = Arrays.asList("word1", "1", "word2", "2", "word2word3", "3", "5", "7");
-
-		actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(word1, word2, word3),
-				Arrays.asList(int1, int2, int3, int4, int5));
-
-		assertEquals(expected, actualList);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
deleted file mode 100644
index 93d1741..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoMapTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.co.CoMapFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoMapTest implements Serializable {
-	private static final long serialVersionUID = 1L;
-
-	private final static class MyCoMap implements CoMapFunction<Double, Integer, String> {
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public String map1(Double value) {
-			return value.toString();
-		}
-
-		@Override
-		public String map2(Integer value) {
-			return value.toString();
-		}
-	}
-
-	@Test
-	public void coMapTest() {
-		CoMapInvokable<Double, Integer, String> invokable = new CoMapInvokable<Double, Integer, String>(new MyCoMap());
-
-		List<String> expectedList = Arrays.asList("1.1", "1", "1.2", "2", "1.3", "3", "1.4", "1.5");
-		List<String> actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(1.1, 1.2, 1.3, 1.4, 1.5), Arrays.asList(1, 2, 3));
-		
-		assertEquals(expectedList, actualList);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
deleted file mode 100755
index 3343ba0..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoStreamReduceTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.flink.streaming.api.function.co.CoReduceFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.junit.Test;
-
-public class CoStreamReduceTest {
-
-	public static class MyCoReduceFunction implements
-			CoReduceFunction<Integer, String, Integer> {
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public Integer reduce1(Integer value1, Integer value2) {
-			return value1 * value2;
-		}
-
-		@Override
-		public String reduce2(String value1, String value2) {
-			return value1 + value2;
-		}
-
-		@Override
-		public Integer map1(Integer value) {
-			return value;
-		}
-
-		@Override
-		public Integer map2(String value) {
-			return Integer.parseInt(value);
-		}
-
-	}
-
-	@Test
-	public void coStreamReduceTest() {
-
-		CoReduceInvokable<Integer, String, Integer> coReduce = new CoReduceInvokable<Integer, String, Integer>(
-				new MyCoReduceFunction());
-
-		List<Integer> expected1 = Arrays.asList(1, 9, 2, 99, 6, 998, 24);
-		List<Integer> result = MockCoContext.createAndExecute(coReduce,
-				Arrays.asList(1, 2, 3, 4), Arrays.asList("9", "9", "8"));
-
-		assertEquals(expected1, result);
-
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
deleted file mode 100644
index 4ab3492..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/CoWindowTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.function.co.CoWindowFunction;
-import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.util.MockCoContext;
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class CoWindowTest {
-
-	public static final class MyCoGroup1 implements CoWindowFunction<Integer, Integer, Integer> {
-
-		private static final long serialVersionUID = 1L;
-
-		@SuppressWarnings("unused")
-		@Override
-		public void coWindow(List<Integer> first, List<Integer> second, Collector<Integer> out)
-				throws Exception {
-			Integer count1 = 0;
-			for (Integer i : first) {
-				count1++;
-			}
-			Integer count2 = 0;
-			for (Integer i : second) {
-				count2++;
-			}
-			out.collect(count1);
-			out.collect(count2);
-
-		}
-
-	}
-
-	public static final class MyCoGroup2 implements
-			CoWindowFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> {
-
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public void coWindow(List<Tuple2<Integer, Integer>> first,
-				List<Tuple2<Integer, Integer>> second, Collector<Integer> out) throws Exception {
-
-			Set<Integer> firstElements = new HashSet<Integer>();
-			for (Tuple2<Integer, Integer> value : first) {
-				firstElements.add(value.f1);
-			}
-			for (Tuple2<Integer, Integer> value : second) {
-				if (firstElements.contains(value.f1)) {
-					out.collect(value.f1);
-				}
-			}
-
-		}
-
-	}
-
-	private static final class MyTS1 implements Timestamp<Integer> {
-
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public long getTimestamp(Integer value) {
-			return value;
-		}
-
-	}
-
-	private static final class MyTS2 implements Timestamp<Tuple2<Integer, Integer>> {
-
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public long getTimestamp(Tuple2<Integer, Integer> value) {
-			return value.f0;
-		}
-
-	}
-
-	@Test
-	public void coWindowGroupReduceTest2() throws Exception {
-
-		CoWindowInvokable<Integer, Integer, Integer> invokable1 = new CoWindowInvokable<Integer, Integer, Integer>(
-				new MyCoGroup1(), 2, 1, new TimestampWrapper<Integer>(new MyTS1(), 1),
-				new TimestampWrapper<Integer>(new MyTS1(), 1));
-
-		// Windowsize 2, slide 1
-		// 1,2|2,3|3,4|4,5
-
-		List<Integer> input11 = new ArrayList<Integer>();
-		input11.add(1);
-		input11.add(1);
-		input11.add(2);
-		input11.add(3);
-		input11.add(3);
-
-		List<Integer> input12 = new ArrayList<Integer>();
-		input12.add(1);
-		input12.add(2);
-		input12.add(3);
-		input12.add(3);
-		input12.add(5);
-
-		// Windows: (1,1,2)(1,1,2)|(2,3,3)(2,3,3)|(3,3)(3,3)|(5)(5)
-		// expected output: 3,2|3,3|2,2|0,1
-
-		List<Integer> expected1 = new ArrayList<Integer>();
-		expected1.add(3);
-		expected1.add(2);
-		expected1.add(3);
-		expected1.add(3);
-		expected1.add(2);
-		expected1.add(2);
-		expected1.add(0);
-		expected1.add(1);
-
-		List<Integer> actual1 = MockCoContext.createAndExecute(invokable1, input11, input12);
-		assertEquals(expected1, actual1);
-
-		CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> invokable2 = new CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer>(
-				new MyCoGroup2(), 2, 3, new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(),
-						1), new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(), 1));
-
-		// WindowSize 2, slide 3
-		// 1,2|4,5|7,8|
-
-		List<Tuple2<Integer, Integer>> input21 = new ArrayList<Tuple2<Integer, Integer>>();
-		input21.add(new Tuple2<Integer, Integer>(1, 1));
-		input21.add(new Tuple2<Integer, Integer>(1, 2));
-		input21.add(new Tuple2<Integer, Integer>(2, 3));
-		input21.add(new Tuple2<Integer, Integer>(3, 4));
-		input21.add(new Tuple2<Integer, Integer>(3, 5));
-		input21.add(new Tuple2<Integer, Integer>(4, 6));
-		input21.add(new Tuple2<Integer, Integer>(4, 7));
-		input21.add(new Tuple2<Integer, Integer>(5, 8));
-
-		List<Tuple2<Integer, Integer>> input22 = new ArrayList<Tuple2<Integer, Integer>>();
-		input22.add(new Tuple2<Integer, Integer>(1, 1));
-		input22.add(new Tuple2<Integer, Integer>(2, 0));
-		input22.add(new Tuple2<Integer, Integer>(2, 2));
-		input22.add(new Tuple2<Integer, Integer>(3, 9));
-		input22.add(new Tuple2<Integer, Integer>(3, 4));
-		input22.add(new Tuple2<Integer, Integer>(4, 10));
-		input22.add(new Tuple2<Integer, Integer>(5, 8));
-		input22.add(new Tuple2<Integer, Integer>(5, 7));
-
-		List<Integer> expected2 = new ArrayList<Integer>();
-		expected2.add(1);
-		expected2.add(2);
-		expected2.add(8);
-		expected2.add(7);
-
-		List<Integer> actual2 = MockCoContext.createAndExecute(invokable2, input21, input22);
-		assertEquals(expected2, actual2);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapTest.java
new file mode 100644
index 0000000..195e67c
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoFlatMapTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.function.co.CoFlatMapFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoFlatMapInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class CoFlatMapTest implements Serializable {
+	private static final long serialVersionUID = 1L;
+
+	private final static class MyCoFlatMap implements CoFlatMapFunction<String, Integer, String> {
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public void flatMap1(String value, Collector<String> coll) {
+			for (int i = 0; i < value.length(); i++) {
+				coll.collect(value.substring(i, i + 1));
+			}
+		}
+
+		@Override
+		public void flatMap2(Integer value, Collector<String> coll) {
+			coll.collect(value.toString());
+		}
+	}
+
+	@Test
+	public void coFlatMapTest() {
+		CoFlatMapInvokable<String, Integer, String> invokable = new CoFlatMapInvokable<String, Integer, String>(
+				new MyCoFlatMap());
+
+		List<String> expectedList = Arrays.asList("a", "b", "c", "1", "d", "e", "f", "2", "g", "h",
+				"e", "3", "4", "5");
+		List<String> actualList = MockCoContext.createAndExecute(invokable,
+				Arrays.asList("abc", "def", "ghe"), Arrays.asList(1, 2, 3, 4, 5));
+
+		assertEquals(expectedList, actualList);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void multipleInputTest() {
+		LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+
+		DataStream<Integer> ds1 = env.fromElements(1, 3, 5);
+		DataStream<Integer> ds2 = env.fromElements(2, 4).merge(ds1);
+		
+		try {
+			ds1.forward().merge(ds2);
+			fail();
+		} catch (RuntimeException e) {
+			// expected
+		}
+		
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceTest.java
new file mode 100644
index 0000000..a531884
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoGroupedReduceTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoGroupedReduceInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.junit.Test;
+
+public class CoGroupedReduceTest {
+
+	private final static class MyCoReduceFunction implements
+			CoReduceFunction<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> {
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public Tuple3<String, String, String> reduce1(Tuple3<String, String, String> value1,
+				Tuple3<String, String, String> value2) {
+			return new Tuple3<String, String, String>(value1.f0, value1.f1 + value2.f1, value1.f2);
+		}
+
+		@Override
+		public Tuple2<Integer, Integer> reduce2(Tuple2<Integer, Integer> value1,
+				Tuple2<Integer, Integer> value2) {
+			return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
+		}
+
+		@Override
+		public String map1(Tuple3<String, String, String> value) {
+			return value.f1;
+		}
+
+		@Override
+		public String map2(Tuple2<Integer, Integer> value) {
+			return value.f1.toString();
+		}
+	}
+
+	@SuppressWarnings("unchecked")
+	@Test
+	public void coGroupedReduceTest() {
+		Tuple3<String, String, String> word1 = new Tuple3<String, String, String>("a", "word1", "b");
+		Tuple3<String, String, String> word2 = new Tuple3<String, String, String>("b", "word2", "a");
+		Tuple3<String, String, String> word3 = new Tuple3<String, String, String>("a", "word3", "a");
+		Tuple2<Integer, Integer> int1 = new Tuple2<Integer, Integer>(2, 1);
+		Tuple2<Integer, Integer> int2 = new Tuple2<Integer, Integer>(1, 2);
+		Tuple2<Integer, Integer> int3 = new Tuple2<Integer, Integer>(0, 3);
+		Tuple2<Integer, Integer> int4 = new Tuple2<Integer, Integer>(2, 4);
+		Tuple2<Integer, Integer> int5 = new Tuple2<Integer, Integer>(1, 5);
+
+		KeySelector<Tuple3<String, String, String>, ?> keySelector0 = new KeySelector<Tuple3<String, String, String>, String>() {
+
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public String getKey(Tuple3<String, String, String> value) throws Exception {
+				return value.f0;
+			}
+		};
+
+		KeySelector<Tuple2<Integer, Integer>, ?> keySelector1 = new KeySelector<Tuple2<Integer, Integer>, Integer>() {
+
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer getKey(Tuple2<Integer, Integer> value) throws Exception {
+				return value.f0;
+			}
+		};
+
+		KeySelector<Tuple3<String, String, String>, ?> keySelector2 = new KeySelector<Tuple3<String, String, String>, String>() {
+
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public String getKey(Tuple3<String, String, String> value) throws Exception {
+				return value.f2;
+			}
+		};
+
+		CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String> invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
+				new MyCoReduceFunction(), keySelector0, keySelector1);
+
+		List<String> expected = Arrays.asList("word1", "1", "word2", "2", "word1word3", "3", "5",
+				"7");
+
+		List<String> actualList = MockCoContext.createAndExecute(invokable,
+				Arrays.asList(word1, word2, word3), Arrays.asList(int1, int2, int3, int4, int5));
+
+		assertEquals(expected, actualList);
+
+		invokable = new CoGroupedReduceInvokable<Tuple3<String, String, String>, Tuple2<Integer, Integer>, String>(
+				new MyCoReduceFunction(), keySelector2, keySelector1);
+
+		expected = Arrays.asList("word1", "1", "word2", "2", "word2word3", "3", "5", "7");
+
+		actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(word1, word2, word3),
+				Arrays.asList(int1, int2, int3, int4, int5));
+
+		assertEquals(expected, actualList);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapTest.java
new file mode 100644
index 0000000..6b84440
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoMapTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.co.CoMapFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoMapInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.junit.Test;
+
+public class CoMapTest implements Serializable {
+	private static final long serialVersionUID = 1L;
+
+	private final static class MyCoMap implements CoMapFunction<Double, Integer, String> {
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public String map1(Double value) {
+			return value.toString();
+		}
+
+		@Override
+		public String map2(Integer value) {
+			return value.toString();
+		}
+	}
+
+	@Test
+	public void coMapTest() {
+		CoMapInvokable<Double, Integer, String> invokable = new CoMapInvokable<Double, Integer, String>(new MyCoMap());
+
+		List<String> expectedList = Arrays.asList("1.1", "1", "1.2", "2", "1.3", "3", "1.4", "1.5");
+		List<String> actualList = MockCoContext.createAndExecute(invokable, Arrays.asList(1.1, 1.2, 1.3, 1.4, 1.5), Arrays.asList(1, 2, 3));
+		
+		assertEquals(expectedList, actualList);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceTest.java
new file mode 100755
index 0000000..7f23fba
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoStreamReduceTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.streaming.api.function.co.CoReduceFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoReduceInvokable;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.junit.Test;
+
+public class CoStreamReduceTest {
+
+	public static class MyCoReduceFunction implements
+			CoReduceFunction<Integer, String, Integer> {
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public Integer reduce1(Integer value1, Integer value2) {
+			return value1 * value2;
+		}
+
+		@Override
+		public String reduce2(String value1, String value2) {
+			return value1 + value2;
+		}
+
+		@Override
+		public Integer map1(Integer value) {
+			return value;
+		}
+
+		@Override
+		public Integer map2(String value) {
+			return Integer.parseInt(value);
+		}
+
+	}
+
+	@Test
+	public void coStreamReduceTest() {
+
+		CoReduceInvokable<Integer, String, Integer> coReduce = new CoReduceInvokable<Integer, String, Integer>(
+				new MyCoReduceFunction());
+
+		List<Integer> expected1 = Arrays.asList(1, 9, 2, 99, 6, 998, 24);
+		List<Integer> result = MockCoContext.createAndExecute(coReduce,
+				Arrays.asList(1, 2, 3, 4), Arrays.asList("9", "9", "8"));
+
+		assertEquals(expected1, result);
+
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowTest.java
new file mode 100644
index 0000000..a4f6120
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/co/CoWindowTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.co;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.function.co.CoWindowFunction;
+import org.apache.flink.streaming.api.invokable.operator.co.CoWindowInvokable;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.util.MockCoContext;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+public class CoWindowTest {
+
+	public static final class MyCoGroup1 implements CoWindowFunction<Integer, Integer, Integer> {
+
+		private static final long serialVersionUID = 1L;
+
+		@SuppressWarnings("unused")
+		@Override
+		public void coWindow(List<Integer> first, List<Integer> second, Collector<Integer> out)
+				throws Exception {
+			Integer count1 = 0;
+			for (Integer i : first) {
+				count1++;
+			}
+			Integer count2 = 0;
+			for (Integer i : second) {
+				count2++;
+			}
+			out.collect(count1);
+			out.collect(count2);
+
+		}
+
+	}
+
+	public static final class MyCoGroup2 implements
+			CoWindowFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public void coWindow(List<Tuple2<Integer, Integer>> first,
+				List<Tuple2<Integer, Integer>> second, Collector<Integer> out) throws Exception {
+
+			Set<Integer> firstElements = new HashSet<Integer>();
+			for (Tuple2<Integer, Integer> value : first) {
+				firstElements.add(value.f1);
+			}
+			for (Tuple2<Integer, Integer> value : second) {
+				if (firstElements.contains(value.f1)) {
+					out.collect(value.f1);
+				}
+			}
+
+		}
+
+	}
+
+	private static final class MyTS1 implements Timestamp<Integer> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public long getTimestamp(Integer value) {
+			return value;
+		}
+
+	}
+
+	private static final class MyTS2 implements Timestamp<Tuple2<Integer, Integer>> {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public long getTimestamp(Tuple2<Integer, Integer> value) {
+			return value.f0;
+		}
+
+	}
+
+	@Test
+	public void coWindowGroupReduceTest2() throws Exception {
+
+		CoWindowInvokable<Integer, Integer, Integer> invokable1 = new CoWindowInvokable<Integer, Integer, Integer>(
+				new MyCoGroup1(), 2, 1, new TimestampWrapper<Integer>(new MyTS1(), 1),
+				new TimestampWrapper<Integer>(new MyTS1(), 1));
+
+		// Windowsize 2, slide 1
+		// 1,2|2,3|3,4|4,5
+
+		List<Integer> input11 = new ArrayList<Integer>();
+		input11.add(1);
+		input11.add(1);
+		input11.add(2);
+		input11.add(3);
+		input11.add(3);
+
+		List<Integer> input12 = new ArrayList<Integer>();
+		input12.add(1);
+		input12.add(2);
+		input12.add(3);
+		input12.add(3);
+		input12.add(5);
+
+		// Windows: (1,1,2)(1,1,2)|(2,3,3)(2,3,3)|(3,3)(3,3)|(5)(5)
+		// expected output: 3,2|3,3|2,2|0,1
+
+		List<Integer> expected1 = new ArrayList<Integer>();
+		expected1.add(3);
+		expected1.add(2);
+		expected1.add(3);
+		expected1.add(3);
+		expected1.add(2);
+		expected1.add(2);
+		expected1.add(0);
+		expected1.add(1);
+
+		List<Integer> actual1 = MockCoContext.createAndExecute(invokable1, input11, input12);
+		assertEquals(expected1, actual1);
+
+		CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer> invokable2 = new CoWindowInvokable<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>, Integer>(
+				new MyCoGroup2(), 2, 3, new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(),
+						1), new TimestampWrapper<Tuple2<Integer, Integer>>(new MyTS2(), 1));
+
+		// WindowSize 2, slide 3
+		// 1,2|4,5|7,8|
+
+		List<Tuple2<Integer, Integer>> input21 = new ArrayList<Tuple2<Integer, Integer>>();
+		input21.add(new Tuple2<Integer, Integer>(1, 1));
+		input21.add(new Tuple2<Integer, Integer>(1, 2));
+		input21.add(new Tuple2<Integer, Integer>(2, 3));
+		input21.add(new Tuple2<Integer, Integer>(3, 4));
+		input21.add(new Tuple2<Integer, Integer>(3, 5));
+		input21.add(new Tuple2<Integer, Integer>(4, 6));
+		input21.add(new Tuple2<Integer, Integer>(4, 7));
+		input21.add(new Tuple2<Integer, Integer>(5, 8));
+
+		List<Tuple2<Integer, Integer>> input22 = new ArrayList<Tuple2<Integer, Integer>>();
+		input22.add(new Tuple2<Integer, Integer>(1, 1));
+		input22.add(new Tuple2<Integer, Integer>(2, 0));
+		input22.add(new Tuple2<Integer, Integer>(2, 2));
+		input22.add(new Tuple2<Integer, Integer>(3, 9));
+		input22.add(new Tuple2<Integer, Integer>(3, 4));
+		input22.add(new Tuple2<Integer, Integer>(4, 10));
+		input22.add(new Tuple2<Integer, Integer>(5, 8));
+		input22.add(new Tuple2<Integer, Integer>(5, 7));
+
+		List<Integer> expected2 = new ArrayList<Integer>();
+		expected2.add(1);
+		expected2.add(2);
+		expected2.add(8);
+		expected2.add(7);
+
+		List<Integer> actual2 = MockCoContext.createAndExecute(invokable2, input21, input22);
+		assertEquals(expected2, actual2);
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java
deleted file mode 100644
index e079e2c..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/BasicWindowBufferTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.util.Collector;
-import org.junit.Test;
-
-public class BasicWindowBufferTest {
-
-	@Test
-	public void testEmitWindow() throws Exception {
-
-		TestCollector<StreamWindow<Integer>> collector = new TestCollector<StreamWindow<Integer>>();
-		List<StreamWindow<Integer>> collected = collector.getCollected();
-
-		WindowBuffer<Integer> wb = new BasicWindowBuffer<Integer>();
-
-		wb.store(2);
-		wb.store(10);
-
-		assertEquals(2, wb.size());
-
-		wb.emitWindow(collector);
-
-		assertEquals(1, collected.size());
-		assertEquals(StreamWindow.fromElements(2, 10), collected.get(0));
-
-		wb.store(4);
-		wb.evict(2);
-
-		assertEquals(1, wb.size());
-
-		wb.emitWindow(collector);
-
-		assertEquals(2, collected.size());
-		assertEquals(StreamWindow.fromElements(4), collected.get(1));
-
-		wb.evict(1);
-
-		wb.emitWindow(collector);
-		assertEquals(2, collected.size());
-	}
-
-	public static class TestCollector<T> implements Collector<T> {
-
-		private final List<T> collected = new ArrayList<T>();
-
-		@Override
-		public void collect(T record) {
-			collected.add(record);
-		}
-
-		@Override
-		public void close() {
-		}
-
-		public List<T> getCollected() {
-			return collected;
-		}
-
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
index 04f09e0..b514d2b 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedStreamDiscretizerTest.java
@@ -26,10 +26,12 @@ import java.util.Set;
 
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.windowbuffer.BasicWindowBuffer;
 import org.apache.flink.streaming.util.MockContext;
 import org.junit.Test;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
index 706b0ef..ebc2677 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamDiscretizerTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.flink.streaming.api.windowing.StreamWindow;
 import org.apache.flink.streaming.api.windowing.helper.Timestamp;
 import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
 import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
@@ -30,6 +31,7 @@ import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
 import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.windowbuffer.BasicWindowBuffer;
 import org.apache.flink.streaming.util.MockContext;
 import org.junit.Test;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
deleted file mode 100644
index 88e90c9..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/StreamWindowTest.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.junit.Test;
-
-public class StreamWindowTest {
-
-	@Test
-	public void creationTest() {
-
-		StreamWindow<Integer> window1 = new StreamWindow<Integer>();
-		assertTrue(window1.isEmpty());
-		assertTrue(window1.windowID != 0);
-
-		window1.add(10);
-		assertEquals(1, window1.size());
-
-		StreamWindow<Integer> window2 = new StreamWindow<Integer>(window1);
-
-		assertTrue(window1.windowID == window2.windowID);
-		assertEquals(1, window2.size());
-
-		StreamWindow<Integer> window3 = new StreamWindow<Integer>(100);
-		assertEquals(100, window3.windowID);
-
-		StreamWindow<Integer> window4 = new StreamWindow<Integer>();
-		assertFalse(window4.windowID == window1.windowID);
-	}
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void mergeTest() throws IOException {
-		StreamWindow<Integer> window1 = new StreamWindow<Integer>().setNumberOfParts(3);
-		StreamWindow<Integer> window2 = new StreamWindow<Integer>(window1.windowID,
-				window1.transformationID, 3);
-		StreamWindow<Integer> window3 = new StreamWindow<Integer>(window1.windowID,
-				window1.transformationID, 3);
-
-		window1.add(1);
-		window2.add(2);
-		window3.add(3);
-
-		Set<Integer> values = new HashSet<Integer>();
-		values.add(1);
-		values.add(2);
-		values.add(3);
-
-		StreamWindow<Integer> merged = StreamWindow.merge(window1, window2, window3);
-
-		assertEquals(3, merged.size());
-		assertEquals(window1.windowID, merged.windowID);
-		assertEquals(values, new HashSet<Integer>(merged));
-
-		try {
-			StreamWindow.merge(window1, new StreamWindow<Integer>());
-			fail();
-		} catch (RuntimeException e) {
-			// good
-		}
-
-		List<StreamWindow<Integer>> wList = merged.split(3);
-
-		StreamWindow<Integer> merged2 = StreamWindow.merge(wList);
-
-		assertEquals(3, merged2.size());
-		assertEquals(window1.windowID, merged2.windowID);
-		assertEquals(values, new HashSet<Integer>(merged2));
-
-	}
-
-	@Test
-	public void serializerTest() throws IOException {
-
-		StreamWindow<Integer> streamWindow = new StreamWindow<Integer>();
-		streamWindow.add(1);
-		streamWindow.add(2);
-		streamWindow.add(3);
-
-		TypeSerializer<StreamWindow<Integer>> ts = new StreamWindowSerializer<Integer>(
-				BasicTypeInfo.INT_TYPE_INFO, null);
-
-		TestOutputView ow = new TestOutputView();
-
-		ts.serialize(streamWindow, ow);
-
-		TestInputView iw = ow.getInputView();
-
-		assertEquals(streamWindow, ts.deserialize(iw));
-
-	}
-
-	@Test
-	public void partitionTest() {
-		StreamWindow<Integer> streamWindow = new StreamWindow<Integer>();
-		streamWindow.add(1);
-		streamWindow.add(2);
-		streamWindow.add(3);
-		streamWindow.add(4);
-		streamWindow.add(5);
-		streamWindow.add(6);
-
-		List<StreamWindow<Integer>> split = streamWindow.split(2);
-		assertEquals(2, split.size());
-		assertEquals(StreamWindow.fromElements(1, 2, 3), split.get(0));
-		assertEquals(StreamWindow.fromElements(4, 5, 6), split.get(1));
-
-		List<StreamWindow<Integer>> split2 = streamWindow.split(6);
-		assertEquals(6, split2.size());
-		assertEquals(StreamWindow.fromElements(1), split2.get(0));
-		assertEquals(StreamWindow.fromElements(2), split2.get(1));
-		assertEquals(StreamWindow.fromElements(3), split2.get(2));
-		assertEquals(StreamWindow.fromElements(4), split2.get(3));
-		assertEquals(StreamWindow.fromElements(5), split2.get(4));
-		assertEquals(StreamWindow.fromElements(6), split2.get(5));
-
-		List<StreamWindow<Integer>> split3 = streamWindow.split(10);
-		assertEquals(6, split3.size());
-		assertEquals(StreamWindow.fromElements(1), split3.get(0));
-		assertEquals(StreamWindow.fromElements(2), split3.get(1));
-		assertEquals(StreamWindow.fromElements(3), split3.get(2));
-		assertEquals(StreamWindow.fromElements(4), split3.get(3));
-		assertEquals(StreamWindow.fromElements(5), split3.get(4));
-		assertEquals(StreamWindow.fromElements(6), split3.get(5));
-
-	}
-
-	private class TestOutputView extends DataOutputStream implements DataOutputView {
-
-		public TestOutputView() {
-			super(new ByteArrayOutputStream(4096));
-		}
-
-		public TestInputView getInputView() {
-			ByteArrayOutputStream baos = (ByteArrayOutputStream) out;
-			return new TestInputView(baos.toByteArray());
-		}
-
-		@Override
-		public void skipBytesToWrite(int numBytes) throws IOException {
-			for (int i = 0; i < numBytes; i++) {
-				write(0);
-			}
-		}
-
-		@Override
-		public void write(DataInputView source, int numBytes) throws IOException {
-			byte[] buffer = new byte[numBytes];
-			source.readFully(buffer);
-			write(buffer);
-		}
-	}
-
-	private class TestInputView extends DataInputStream implements DataInputView {
-
-		public TestInputView(byte[] data) {
-			super(new ByteArrayInputStream(data));
-		}
-
-		@Override
-		public void skipBytesToRead(int numBytes) throws IOException {
-			while (numBytes > 0) {
-				int skipped = skipBytes(numBytes);
-				numBytes -= skipped;
-			}
-		}
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java
deleted file mode 100644
index 8287202..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingGroupedPreReducerTest.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.operators.Keys;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.invokable.operator.windowing.BasicWindowBufferTest.TestCollector;
-import org.apache.flink.streaming.util.keys.KeySelectorUtil;
-import org.junit.Test;
-
-public class TumblingGroupedPreReducerTest {
-
-	TypeInformation<Tuple2<Integer, Integer>> type = TypeExtractor
-			.getForObject(new Tuple2<Integer, Integer>(1, 1));
-	TypeSerializer<Tuple2<Integer, Integer>> serializer = type.createSerializer(null);
-
-	KeySelector<Tuple2<Integer, Integer>, ?> key = KeySelectorUtil.getSelectorForKeys(
-			new Keys.ExpressionKeys<Tuple2<Integer, Integer>>(new int[] { 0 }, type), type, null);
-
-	Reducer reducer = new Reducer();
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void testEmitWindow() throws Exception {
-
-		List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
-		inputs.add(new Tuple2<Integer, Integer>(1, 1));
-		inputs.add(new Tuple2<Integer, Integer>(0, 0));
-		inputs.add(new Tuple2<Integer, Integer>(1, -1));
-		inputs.add(new Tuple2<Integer, Integer>(1, -2));
-
-		TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector = new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
-		List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
-
-		WindowBuffer<Tuple2<Integer, Integer>> wb = new TumblingGroupedPreReducer<Tuple2<Integer, Integer>>(
-				reducer, key, serializer);
-
-		wb.store(serializer.copy(inputs.get(0)));
-		wb.store(serializer.copy(inputs.get(1)));
-
-		assertEquals(2, wb.size());
-
-		wb.emitWindow(collector);
-
-		assertEquals(1, collected.size());
-
-		assertSetEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(1, 1),
-				new Tuple2<Integer, Integer>(0, 0)), collected.get(0));
-
-		// Test automatic eviction
-		assertEquals(0, wb.size());
-
-		wb.store(serializer.copy(inputs.get(0)));
-		wb.store(serializer.copy(inputs.get(1)));
-		wb.store(serializer.copy(inputs.get(2)));
-
-		// Nothing should happen here
-		wb.evict(3);
-
-		wb.store(serializer.copy(inputs.get(3)));
-
-		assertEquals(4, wb.size());
-		wb.emitWindow(collector);
-
-		assertEquals(2, collected.size());
-
-		assertSetEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(3, -2),
-				new Tuple2<Integer, Integer>(0, 0)), collected.get(1));
-
-		// Test whether function is mutating inputs or not
-		assertEquals(2, reducer.allInputs.size());
-		assertEquals(reducer.allInputs.get(0), inputs.get(2));
-		assertEquals(reducer.allInputs.get(1), inputs.get(3));
-
-	}
-
-	private static <T> void assertSetEquals(Collection<T> first, Collection<T> second) {
-		assertEquals(new HashSet<T>(first), new HashSet<T>(second));
-	}
-
-	@SuppressWarnings("serial")
-	private class Reducer implements ReduceFunction<Tuple2<Integer, Integer>> {
-
-		public List<Tuple2<Integer, Integer>> allInputs = new ArrayList<Tuple2<Integer, Integer>>();
-
-		@Override
-		public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1,
-				Tuple2<Integer, Integer> value2) throws Exception {
-			allInputs.add(value2);
-			value1.f0 = value1.f0 + value2.f0;
-			value1.f1 = value1.f1 + value2.f1;
-			return value1;
-		}
-
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/bb5dc7e3/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java
deleted file mode 100644
index 6e29e68..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/TumblingPreReducerTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator.windowing;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.invokable.operator.windowing.BasicWindowBufferTest.TestCollector;
-import org.junit.Test;
-
-public class TumblingPreReducerTest {
-
-	TypeSerializer<Tuple2<Integer, Integer>> serializer = TypeExtractor.getForObject(
-			new Tuple2<Integer, Integer>(1, 1)).createSerializer(null);
-
-	Reducer reducer = new Reducer();
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void testEmitWindow() throws Exception {
-
-		List<Tuple2<Integer, Integer>> inputs = new ArrayList<Tuple2<Integer, Integer>>();
-		inputs.add(new Tuple2<Integer, Integer>(1, 1));
-		inputs.add(new Tuple2<Integer, Integer>(2, 0));
-		inputs.add(new Tuple2<Integer, Integer>(3, -1));
-		inputs.add(new Tuple2<Integer, Integer>(4, -2));
-
-		TestCollector<StreamWindow<Tuple2<Integer, Integer>>> collector = new TestCollector<StreamWindow<Tuple2<Integer, Integer>>>();
-		List<StreamWindow<Tuple2<Integer, Integer>>> collected = collector.getCollected();
-
-		WindowBuffer<Tuple2<Integer, Integer>> wb = new TumblingPreReducer<Tuple2<Integer, Integer>>(
-				reducer, serializer);
-
-		wb.store(serializer.copy(inputs.get(0)));
-		wb.store(serializer.copy(inputs.get(1)));
-
-		assertEquals(2, wb.size());
-
-		wb.emitWindow(collector);
-
-		assertEquals(1, collected.size());
-		assertEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(3, 1)),
-				collected.get(0));
-
-		// Test automatic eviction
-		assertEquals(0, wb.size());
-
-		wb.store(serializer.copy(inputs.get(0)));
-		wb.store(serializer.copy(inputs.get(1)));
-		wb.store(serializer.copy(inputs.get(2)));
-
-		// Nothing should happen here
-		wb.evict(3);
-
-		wb.store(serializer.copy(inputs.get(3)));
-
-		assertEquals(4, wb.size());
-		wb.emitWindow(collector);
-
-		assertEquals(2, collected.size());
-		assertEquals(StreamWindow.fromElements(new Tuple2<Integer, Integer>(10, -2)),
-				collected.get(1));
-
-		// Test whether function is mutating inputs or not
-		assertEquals(4, reducer.allInputs.size());
-		assertEquals(reducer.allInputs.get(0), inputs.get(1));
-		assertEquals(reducer.allInputs.get(1), inputs.get(1));
-		assertEquals(reducer.allInputs.get(2), inputs.get(2));
-		assertEquals(reducer.allInputs.get(3), inputs.get(3));
-
-	}
-
-	@SuppressWarnings("serial")
-	private class Reducer implements ReduceFunction<Tuple2<Integer, Integer>> {
-
-		public List<Tuple2<Integer, Integer>> allInputs = new ArrayList<Tuple2<Integer, Integer>>();
-
-		@Override
-		public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1,
-				Tuple2<Integer, Integer> value2) throws Exception {
-			allInputs.add(value2);
-			value1.f0 = value1.f0 + value2.f0;
-			value1.f1 = value1.f1 + value2.f1;
-			return value1;
-		}
-
-	}
-
-}


[03/20] flink git commit: [FLINK-1176] [streaming] WindowedDataStream rework for new windowing runtime

Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
deleted file mode 100644
index f38d5c1..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/GroupedWindowInvokableTest.java
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
-import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class GroupedWindowInvokableTest {
-
-	KeySelector<Tuple2<Integer, String>, ?> keySelector = new KeySelector<Tuple2<Integer, String>, String>() {
-
-		private static final long serialVersionUID = 1L;
-
-		@Override
-		public String getKey(Tuple2<Integer, String> value) throws Exception {
-			return value.f1;
-		}
-	};
-
-	/**
-	 * Tests that illegal arguments result in failure. The following cases are
-	 * tested: 1) having no trigger 2) having no eviction 3) having neither
-	 * eviction nor trigger 4) having both, central and distributed eviction.
-	 */
-	@Test
-	public void testGroupedWindowInvokableFailTest() {
-
-		// create dummy reduce function
-		ReduceFunction<Object> userFunction = new ReduceFunction<Object>() {
-
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Object reduce(Object value1, Object value2) throws Exception {
-				return null;
-			}
-		};
-
-		// create dummy keySelector
-		KeySelector<Object, Object> keySelector = new KeySelector<Object, Object>() {
-
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Object getKey(Object value) throws Exception {
-				return null;
-			}
-		};
-
-		// create policy lists
-		LinkedList<CloneableEvictionPolicy<Object>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<Object>>();
-		LinkedList<CloneableTriggerPolicy<Object>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<Object>>();
-		LinkedList<EvictionPolicy<Object>> centralEvictionPolicies = new LinkedList<EvictionPolicy<Object>>();
-		LinkedList<TriggerPolicy<Object>> centralTriggerPolicies = new LinkedList<TriggerPolicy<Object>>();
-
-		// empty trigger and policy lists should fail
-		try {
-			new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (1)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// null for trigger and policy lists should fail
-		try {
-			new GroupedWindowInvokable<Object, Object>(userFunction, keySelector, null, null, null,
-					null);
-			fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (2)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// empty eviction should still fail
-		centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
-		distributedTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
-		try {
-			new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance without any eviction policy should cause an UnsupportedOperationException but didn't. (3)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// empty trigger should still fail
-		centralTriggerPolicies.clear();
-		distributedTriggerPolicies.clear();
-		centralEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
-		try {
-			new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance without any trigger policy should cause an UnsupportedOperationException but didn't. (4)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-		// having both, central and distributed eviction, at the same time
-		// should fail
-		centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
-		distributedEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
-		try {
-			new GroupedWindowInvokable<Object, Object>(userFunction, keySelector,
-					distributedTriggerPolicies, distributedEvictionPolicies,
-					centralTriggerPolicies, centralEvictionPolicies);
-			fail("Creating instance with central and distributed eviction should cause an UnsupportedOperationException but didn't. (4)");
-		} catch (UnsupportedOperationException e) {
-			// that's the expected case
-		}
-
-	}
-
-	/**
-	 * Test for not active distributed triggers with single field
-	 */
-	@Test
-	public void testGroupedWindowInvokableDistributedTriggerSimple() {
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(1);
-		inputs.add(5);
-		inputs.add(5);
-		inputs.add(5);
-		inputs.add(1);
-		inputs.add(1);
-		inputs.add(5);
-		inputs.add(1);
-		inputs.add(5);
-
-		List<Integer> expectedDistributedEviction = new ArrayList<Integer>();
-		expectedDistributedEviction.add(15);
-		expectedDistributedEviction.add(3);
-		expectedDistributedEviction.add(3);
-		expectedDistributedEviction.add(15);
-
-		List<Integer> expectedCentralEviction = new ArrayList<Integer>();
-		expectedCentralEviction.add(2);
-		expectedCentralEviction.add(5);
-		expectedCentralEviction.add(15);
-		expectedCentralEviction.add(2);
-		expectedCentralEviction.add(5);
-		expectedCentralEviction.add(2);
-		expectedCentralEviction.add(5);
-		expectedCentralEviction.add(1);
-		expectedCentralEviction.add(5);
-
-		LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-		// Trigger on every 2nd element, but the first time after the 3rd
-		triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
-		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
-		// On every 2nd element, remove the oldest 2 elements, but the first
-		// time after the 3rd element
-		evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
-		LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
-
-		ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer getKey(Integer value) {
-				return value;
-			}
-		};
-
-		GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
-				reduceFunction, keySelector, triggers, evictions, centralTriggers, null);
-
-		List<Integer> result = MockContext.createAndExecute(invokable, inputs);
-
-		List<Integer> actual = new LinkedList<Integer>();
-		for (Integer current : result) {
-			actual.add(current);
-		}
-
-		assertEquals(new HashSet<Integer>(expectedDistributedEviction),
-				new HashSet<Integer>(actual));
-		assertEquals(expectedDistributedEviction.size(), actual.size());
-
-		// Run test with central eviction
-		triggers.clear();
-		centralTriggers.add(new CountTriggerPolicy<Integer>(2, -1));
-		LinkedList<EvictionPolicy<Integer>> centralEvictions = new LinkedList<EvictionPolicy<Integer>>();
-		centralEvictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
-		invokable = new GroupedWindowInvokable<Integer, Integer>(reduceFunction, keySelector,
-				triggers, null, centralTriggers, centralEvictions);
-
-		result = MockContext.createAndExecute(invokable, inputs);
-		actual = new LinkedList<Integer>();
-		for (Integer current : result) {
-			actual.add(current);
-		}
-
-		assertEquals(new HashSet<Integer>(expectedCentralEviction), new HashSet<Integer>(actual));
-		assertEquals(expectedCentralEviction.size(), actual.size());
-	}
-
-	/**
-	 * Test for non active distributed triggers with separated key field
-	 */
-	@Test
-	public void testGroupedWindowInvokableDistributedTriggerComplex() {
-		List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
-		inputs2.add(new Tuple2<Integer, String>(1, "a"));
-		inputs2.add(new Tuple2<Integer, String>(0, "b"));
-		inputs2.add(new Tuple2<Integer, String>(2, "a"));
-		inputs2.add(new Tuple2<Integer, String>(-1, "a"));
-		inputs2.add(new Tuple2<Integer, String>(-2, "a"));
-		inputs2.add(new Tuple2<Integer, String>(10, "a"));
-		inputs2.add(new Tuple2<Integer, String>(2, "b"));
-		inputs2.add(new Tuple2<Integer, String>(1, "a"));
-
-		List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
-		expected2.add(new Tuple2<Integer, String>(-1, "a"));
-		expected2.add(new Tuple2<Integer, String>(-2, "a"));
-		expected2.add(new Tuple2<Integer, String>(0, "b"));
-
-		LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
-		// Trigger on every 2nd element, but the first time after the 3rd
-		triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
-
-		LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
-		// On every 2nd element, remove the oldest 2 elements, but the first
-		// time after the 3rd element
-		evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
-
-		LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
-
-		GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
-				new ReduceFunction<Tuple2<Integer, String>>() {
-					private static final long serialVersionUID = 1L;
-
-					@Override
-					public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
-							Tuple2<Integer, String> value2) throws Exception {
-						if (value1.f0 <= value2.f0) {
-							return value1;
-						} else {
-							return value2;
-						}
-					}
-				}, keySelector, triggers, evictions, centralTriggers, null);
-
-		List<Tuple2<Integer, String>> result = MockContext.createAndExecute(invokable2, inputs2);
-
-		List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
-		for (Tuple2<Integer, String> current : result) {
-			actual2.add(current);
-		}
-
-		assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
-				new HashSet<Tuple2<Integer, String>>(actual2));
-		assertEquals(expected2.size(), actual2.size());
-	}
-
-	/**
-	 * Test for active centralized trigger
-	 */
-	@Test
-	public void testGroupedWindowInvokableCentralActiveTrigger() {
-
-		List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
-		inputs.add(new Tuple2<Integer, String>(1, "a"));
-		inputs.add(new Tuple2<Integer, String>(1, "b"));
-		inputs.add(new Tuple2<Integer, String>(1, "c"));
-		inputs.add(new Tuple2<Integer, String>(2, "a"));
-		inputs.add(new Tuple2<Integer, String>(2, "b"));
-		inputs.add(new Tuple2<Integer, String>(2, "c"));
-		inputs.add(new Tuple2<Integer, String>(2, "b"));
-		inputs.add(new Tuple2<Integer, String>(2, "a"));
-		inputs.add(new Tuple2<Integer, String>(2, "c"));
-		inputs.add(new Tuple2<Integer, String>(3, "c"));
-		inputs.add(new Tuple2<Integer, String>(3, "a"));
-		inputs.add(new Tuple2<Integer, String>(3, "b"));
-		inputs.add(new Tuple2<Integer, String>(4, "a"));
-		inputs.add(new Tuple2<Integer, String>(4, "b"));
-		inputs.add(new Tuple2<Integer, String>(4, "c"));
-		inputs.add(new Tuple2<Integer, String>(5, "c"));
-		inputs.add(new Tuple2<Integer, String>(5, "a"));
-		inputs.add(new Tuple2<Integer, String>(5, "b"));
-		inputs.add(new Tuple2<Integer, String>(10, "b"));
-		inputs.add(new Tuple2<Integer, String>(10, "a"));
-		inputs.add(new Tuple2<Integer, String>(10, "c"));
-		inputs.add(new Tuple2<Integer, String>(11, "a"));
-		inputs.add(new Tuple2<Integer, String>(11, "a"));
-		inputs.add(new Tuple2<Integer, String>(11, "c"));
-		inputs.add(new Tuple2<Integer, String>(11, "c"));
-		inputs.add(new Tuple2<Integer, String>(11, "b"));
-		inputs.add(new Tuple2<Integer, String>(11, "b"));
-
-		// Expected result:
-		// For each group (a,b and c):
-		// 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
-		// 12-12-5-10-32
-
-		List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
-		expected.add(new Tuple2<Integer, String>(12, "a"));
-		expected.add(new Tuple2<Integer, String>(12, "b"));
-		expected.add(new Tuple2<Integer, String>(12, "c"));
-		expected.add(new Tuple2<Integer, String>(12, "a"));
-		expected.add(new Tuple2<Integer, String>(12, "b"));
-		expected.add(new Tuple2<Integer, String>(12, "c"));
-		expected.add(new Tuple2<Integer, String>(5, "a"));
-		expected.add(new Tuple2<Integer, String>(5, "b"));
-		expected.add(new Tuple2<Integer, String>(5, "c"));
-		expected.add(new Tuple2<Integer, String>(10, "a"));
-		expected.add(new Tuple2<Integer, String>(10, "b"));
-		expected.add(new Tuple2<Integer, String>(10, "c"));
-		expected.add(new Tuple2<Integer, String>(32, "a"));
-		expected.add(new Tuple2<Integer, String>(32, "b"));
-		expected.add(new Tuple2<Integer, String>(32, "c"));
-
-		Timestamp<Tuple2<Integer, String>> myTimeStamp = new Timestamp<Tuple2<Integer, String>>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public long getTimestamp(Tuple2<Integer, String> value) {
-				return value.f0;
-			}
-		};
-
-		TimestampWrapper<Tuple2<Integer, String>> myTimeStampWrapper = new TimestampWrapper<Tuple2<Integer, String>>(
-				myTimeStamp, 1);
-
-		ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
-					Tuple2<Integer, String> value2) throws Exception {
-				return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
-			}
-		};
-
-		LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
-		// Trigger every 2 time units but delay the first trigger by 2 (First
-		// trigger after 4, then every 2)
-		triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
-
-		LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
-		// Always delete all elements older then 4
-		evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
-
-		LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
-
-		GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
-				myReduceFunction, keySelector, distributedTriggers, evictions, triggers, null);
-
-		ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
-		for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
-				new HashSet<Tuple2<Integer, String>>(result));
-		assertEquals(expected.size(), result.size());
-
-		// repeat the test with central eviction. The result should be the same.
-		triggers.clear();
-		triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
-		evictions.clear();
-		LinkedList<EvictionPolicy<Tuple2<Integer, String>>> centralEvictions = new LinkedList<EvictionPolicy<Tuple2<Integer, String>>>();
-		centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
-
-		invokable = new GroupedWindowInvokable<Tuple2<Integer, String>, Tuple2<Integer, String>>(
-				myReduceFunction, keySelector, distributedTriggers, evictions, triggers,
-				centralEvictions);
-
-		result = new ArrayList<Tuple2<Integer, String>>();
-		for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
-				new HashSet<Tuple2<Integer, String>>(result));
-		assertEquals(expected.size(), result.size());
-	}
-
-	/**
-	 * Test for multiple centralized trigger
-	 */
-	@Test
-	public void testGroupedWindowInvokableMultipleCentralTrigger() {
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		triggers.add(new CountTriggerPolicy<Integer>(8));
-		triggers.add(new CountTriggerPolicy<Integer>(5));
-
-		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
-		// The active wrapper causes eviction even on (fake) elements which
-		// triggered, but does not belong to the group.
-		evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
-				new TumblingEvictionPolicy<Integer>()));
-
-		LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(2);
-		inputs.add(2);
-		inputs.add(2);
-		inputs.add(1);
-		// 1st Trigger: 2;6
-		inputs.add(2);
-		inputs.add(1);
-		inputs.add(2);
-		// 2nd Trigger: 1;4
-		inputs.add(2);
-		inputs.add(1);
-		// Final: 1,2
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(2);
-		expected.add(6);
-		expected.add(4);
-		expected.add(1);
-		expected.add(2);
-		expected.add(1);
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
-				myReduceFunction, new KeySelector<Integer, Integer>() {
-					private static final long serialVersionUID = 1L;
-
-					@Override
-					public Integer getKey(Integer value) {
-						return value;
-					}
-				}, distributedTriggers, evictions, triggers, null);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
-		assertEquals(expected.size(), result.size());
-	}
-
-	/**
-	 * Test for combination of centralized trigger and distributed trigger at
-	 * the same time
-	 */
-	@Test
-	public void testGroupedWindowInvokableCentralAndDistrTrigger() {
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		triggers.add(new CountTriggerPolicy<Integer>(8));
-		triggers.add(new CountTriggerPolicy<Integer>(5));
-
-		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
-		// The active wrapper causes eviction even on (fake) elements which
-		// triggered, but does not belong to the group.
-		evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
-				new TumblingEvictionPolicy<Integer>()));
-
-		LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
-		distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(2);
-		inputs.add(2);
-		// local on 2 => 4
-		inputs.add(2);
-		inputs.add(1);
-		// local on 1 => 2
-		// and 1st Central: 2;2
-		// SUMS up to 2;2
-		inputs.add(2);
-		// local on 2 => 2
-		inputs.add(1);
-		inputs.add(2);
-		// 2nd Central: 1;2
-		inputs.add(2);
-		inputs.add(1);
-		// Final: 1,2
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(4);
-		expected.add(2);
-		expected.add(2);
-		expected.add(2);
-		expected.add(1);
-		expected.add(2);
-		expected.add(1);
-		expected.add(2);
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		GroupedWindowInvokable<Integer, Integer> invokable = new GroupedWindowInvokable<Integer, Integer>(
-				myReduceFunction, new KeySelector<Integer, Integer>() {
-					private static final long serialVersionUID = 1L;
-
-					@Override
-					public Integer getKey(Integer value) {
-						return value;
-					}
-				}, distributedTriggers, evictions, triggers, null);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
-		assertEquals(expected.size(), result.size());
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
deleted file mode 100644
index 83b4596..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/WindowInvokableTest.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.invokable.operator;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.streaming.api.windowing.helper.Timestamp;
-import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
-import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
-import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
-import org.apache.flink.streaming.util.MockContext;
-import org.junit.Test;
-
-public class WindowInvokableTest {
-
-	/**
-	 * Test case equal to {@link WindowReduceInvokableTest}
-	 */
-	@Test
-	public void testWindowInvokableWithTimePolicy() {
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		inputs.add(1);
-		inputs.add(2);
-		inputs.add(2);
-		inputs.add(3);
-		inputs.add(4);
-		inputs.add(5);
-		inputs.add(10);
-		inputs.add(11);
-		inputs.add(11);
-		// 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
-		// 12-12-5-10-32
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(12);
-		expected.add(12);
-		expected.add(5);
-		expected.add(10);
-		expected.add(32);
-
-		Timestamp<Integer> myTimeStamp = new Timestamp<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public long getTimestamp(Integer value) {
-				return value;
-			}
-		};
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		// Trigger every 2 time units but delay the first trigger by 2 (First
-		// trigger after 4, then every 2)
-		triggers.add(new TimeTriggerPolicy<Integer>(2L, new TimestampWrapper<Integer>(myTimeStamp,
-				1), 2L));
-
-		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
-		// Always delete all elements older then 4
-		evictions.add(new TimeEvictionPolicy<Integer>(4L, new TimestampWrapper<Integer>(
-				myTimeStamp, 1)));
-
-		WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(expected, result);
-	}
-
-	/**
-	 * Test case equal to {@link BatchReduceTest}
-	 */
-	@Test
-	public void testWindowInvokableWithCountPolicy() {
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		for (Integer i = 1; i <= 10; i++) {
-			inputs.add(i);
-		}
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		/*
-		 * The following setup reassembles the batch size 3 and the slide size 2
-		 * of the BatchReduceInvokable.
-		 */
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		// Trigger on every 2nd element, but the first time after the 3rd
-		triggers.add(new CountTriggerPolicy<Integer>(2, -1));
-
-		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
-		// On every 2nd element, remove the oldest 2 elements, but the first
-		// time after the 3rd element
-		evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
-
-		WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(6);
-		expected.add(12);
-		expected.add(18);
-		expected.add(24);
-		expected.add(19);
-		List<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-		assertEquals(expected, result);
-
-		/*
-		 * Begin test part 2
-		 */
-
-		List<Integer> inputs2 = new ArrayList<Integer>();
-		inputs2.add(1);
-		inputs2.add(2);
-		inputs2.add(-5); // changed this value to make sure it is excluded from
-							// the result
-		inputs2.add(-3);
-		inputs2.add(-4);
-
-		myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				if (value1 <= value2) {
-					return value1;
-				} else {
-					return value2;
-				}
-			};
-		};
-
-		/*
-		 * The following setup reassembles the batch size 2 and the slide size 3
-		 * of the BatchReduceInvokable.
-		 */
-		triggers = new LinkedList<TriggerPolicy<Integer>>();
-		// Trigger after every 3rd element, but the first time after the 2nd
-		triggers.add(new CountTriggerPolicy<Integer>(3, 1));
-
-		evictions = new LinkedList<EvictionPolicy<Integer>>();
-		// On every 3rd element, remove the oldest 3 elements, but the first
-		// time after on the 5th element
-		evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
-
-		WindowInvokable<Integer, Integer> invokable2 = new WindowReduceInvokable<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		List<Integer> expected2 = new ArrayList<Integer>();
-		expected2.add(1);
-		expected2.add(-4);
-
-		result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable2, inputs2)) {
-			result.add(t);
-		}
-
-		assertEquals(expected2, result);
-
-	}
-
-	@Test
-	public void testWindowInvokableWithMultiplePolicies() {
-		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
-		triggers.add(new CountTriggerPolicy<Integer>(2));
-		triggers.add(new CountTriggerPolicy<Integer>(3));
-
-		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
-		evictions.add(new CountEvictionPolicy<Integer>(2, 2));
-		evictions.add(new CountEvictionPolicy<Integer>(3, 3));
-
-		List<Integer> inputs = new ArrayList<Integer>();
-		for (Integer i = 1; i <= 10; i++) {
-			inputs.add(i);
-		}
-		/**
-		 * <code>
-		 * VAL: 1,2,3,4,5,6,7,8,9,10
-		 * TR1:   |   |   |   |   |
-		 * TR2:     |     |     |
-		 * EV1:   2   2   2   2   2
-		 * EV2:     3     3     3
-		 * </code>
-		 */
-
-		List<Integer> expected = new ArrayList<Integer>();
-		expected.add(3);
-		expected.add(3);
-		expected.add(4);
-		expected.add(11);
-		expected.add(15);
-		expected.add(9);
-		expected.add(10);
-
-		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
-			private static final long serialVersionUID = 1L;
-
-			@Override
-			public Integer reduce(Integer value1, Integer value2) throws Exception {
-				return value1 + value2;
-			}
-		};
-
-		WindowInvokable<Integer, Integer> invokable = new WindowReduceInvokable<Integer>(
-				myReduceFunction, triggers, evictions);
-
-		ArrayList<Integer> result = new ArrayList<Integer>();
-		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
-			result.add(t);
-		}
-
-		assertEquals(expected, result);
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java
new file mode 100644
index 0000000..1bbdcf6
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/GroupedWindowInvokableTest.java
@@ -0,0 +1,574 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.api.windowing.policy.ActiveCloneableEvictionPolicyWrapper;
+import org.apache.flink.streaming.api.windowing.policy.CloneableEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CloneableTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TumblingEvictionPolicy;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class GroupedWindowInvokableTest {
+
+	KeySelector<Tuple2<Integer, String>, ?> keySelector = new KeySelector<Tuple2<Integer, String>, String>() {
+
+		private static final long serialVersionUID = 1L;
+
+		@Override
+		public String getKey(Tuple2<Integer, String> value) throws Exception {
+			return value.f1;
+		}
+	};
+
+	/**
+	 * Tests that illegal arguments result in failure. The following cases are
+	 * tested: 1) having no trigger 2) having no eviction 3) having neither
+	 * eviction nor trigger 4) having both, central and distributed eviction.
+	 */
+	@Test
+	public void testGroupedWindowInvokableFailTest() {
+
+		// create dummy reduce function
+		ReduceFunction<Object> userFunction = new ReduceFunction<Object>() {
+
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Object reduce(Object value1, Object value2) throws Exception {
+				return null;
+			}
+		};
+
+		// create dummy keySelector
+		KeySelector<Object, Object> keySelector = new KeySelector<Object, Object>() {
+
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Object getKey(Object value) throws Exception {
+				return null;
+			}
+		};
+
+		// create policy lists
+		LinkedList<CloneableEvictionPolicy<Object>> distributedEvictionPolicies = new LinkedList<CloneableEvictionPolicy<Object>>();
+		LinkedList<CloneableTriggerPolicy<Object>> distributedTriggerPolicies = new LinkedList<CloneableTriggerPolicy<Object>>();
+		LinkedList<EvictionPolicy<Object>> centralEvictionPolicies = new LinkedList<EvictionPolicy<Object>>();
+		LinkedList<TriggerPolicy<Object>> centralTriggerPolicies = new LinkedList<TriggerPolicy<Object>>();
+
+		// empty trigger and policy lists should fail
+		try {
+			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
+					distributedTriggerPolicies, distributedEvictionPolicies,
+					centralTriggerPolicies, centralEvictionPolicies);
+			fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (1)");
+		} catch (UnsupportedOperationException e) {
+			// that's the expected case
+		}
+
+		// null for trigger and policy lists should fail
+		try {
+			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector, null, null, null,
+					null);
+			fail("Creating instance without any trigger or eviction policy should cause an UnsupportedOperationException but didn't. (2)");
+		} catch (UnsupportedOperationException e) {
+			// that's the expected case
+		}
+
+		// empty eviction should still fail
+		centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+		distributedTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+		try {
+			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
+					distributedTriggerPolicies, distributedEvictionPolicies,
+					centralTriggerPolicies, centralEvictionPolicies);
+			fail("Creating instance without any eviction policy should cause an UnsupportedOperationException but didn't. (3)");
+		} catch (UnsupportedOperationException e) {
+			// that's the expected case
+		}
+
+		// empty trigger should still fail
+		centralTriggerPolicies.clear();
+		distributedTriggerPolicies.clear();
+		centralEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
+		try {
+			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
+					distributedTriggerPolicies, distributedEvictionPolicies,
+					centralTriggerPolicies, centralEvictionPolicies);
+			fail("Creating instance without any trigger policy should cause an UnsupportedOperationException but didn't. (4)");
+		} catch (UnsupportedOperationException e) {
+			// that's the expected case
+		}
+
+		// having both, central and distributed eviction, at the same time
+		// should fail
+		centralTriggerPolicies.add(new CountTriggerPolicy<Object>(5));
+		distributedEvictionPolicies.add(new CountEvictionPolicy<Object>(5));
+		try {
+			new GroupedStreamDiscretizer<Object, Object>(userFunction, keySelector,
+					distributedTriggerPolicies, distributedEvictionPolicies,
+					centralTriggerPolicies, centralEvictionPolicies);
+			fail("Creating instance with central and distributed eviction should cause an UnsupportedOperationException but didn't. (4)");
+		} catch (UnsupportedOperationException e) {
+			// that's the expected case
+		}
+
+	}
+
+	/**
+	 * Test for not active distributed triggers with single field
+	 */
+	@Test
+	public void testGroupedWindowInvokableDistributedTriggerSimple() {
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(1);
+		inputs.add(5);
+		inputs.add(5);
+		inputs.add(5);
+		inputs.add(1);
+		inputs.add(1);
+		inputs.add(5);
+		inputs.add(1);
+		inputs.add(5);
+
+		List<Integer> expectedDistributedEviction = new ArrayList<Integer>();
+		expectedDistributedEviction.add(15);
+		expectedDistributedEviction.add(3);
+		expectedDistributedEviction.add(3);
+		expectedDistributedEviction.add(15);
+
+		List<Integer> expectedCentralEviction = new ArrayList<Integer>();
+		expectedCentralEviction.add(2);
+		expectedCentralEviction.add(5);
+		expectedCentralEviction.add(15);
+		expectedCentralEviction.add(2);
+		expectedCentralEviction.add(5);
+		expectedCentralEviction.add(2);
+		expectedCentralEviction.add(5);
+		expectedCentralEviction.add(1);
+		expectedCentralEviction.add(5);
+
+		LinkedList<CloneableTriggerPolicy<Integer>> triggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+		// Trigger on every 2nd element, but the first time after the 3rd
+		triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+		// On every 2nd element, remove the oldest 2 elements, but the first
+		// time after the 3rd element
+		evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+		LinkedList<TriggerPolicy<Integer>> centralTriggers = new LinkedList<TriggerPolicy<Integer>>();
+
+		ReduceFunction<Integer> reduceFunction = new ReduceFunction<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer reduce(Integer value1, Integer value2) throws Exception {
+				return value1 + value2;
+			}
+		};
+
+		KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer getKey(Integer value) {
+				return value;
+			}
+		};
+
+		GroupedStreamDiscretizer<Integer, Integer> invokable = new GroupedStreamDiscretizer<Integer, Integer>(
+				reduceFunction, keySelector, triggers, evictions, centralTriggers, null);
+
+		List<Integer> result = MockContext.createAndExecute(invokable, inputs);
+
+		List<Integer> actual = new LinkedList<Integer>();
+		for (Integer current : result) {
+			actual.add(current);
+		}
+
+		assertEquals(new HashSet<Integer>(expectedDistributedEviction),
+				new HashSet<Integer>(actual));
+		assertEquals(expectedDistributedEviction.size(), actual.size());
+
+		// Run test with central eviction
+		triggers.clear();
+		centralTriggers.add(new CountTriggerPolicy<Integer>(2, -1));
+		LinkedList<EvictionPolicy<Integer>> centralEvictions = new LinkedList<EvictionPolicy<Integer>>();
+		centralEvictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+		invokable = new GroupedStreamDiscretizer<Integer, Integer>(reduceFunction, keySelector,
+				triggers, null, centralTriggers, centralEvictions);
+
+		result = MockContext.createAndExecute(invokable, inputs);
+		actual = new LinkedList<Integer>();
+		for (Integer current : result) {
+			actual.add(current);
+		}
+
+		assertEquals(new HashSet<Integer>(expectedCentralEviction), new HashSet<Integer>(actual));
+		assertEquals(expectedCentralEviction.size(), actual.size());
+	}
+
+	/**
+	 * Test for non active distributed triggers with separated key field
+	 */
+	@Test
+	public void testGroupedWindowInvokableDistributedTriggerComplex() {
+		List<Tuple2<Integer, String>> inputs2 = new ArrayList<Tuple2<Integer, String>>();
+		inputs2.add(new Tuple2<Integer, String>(1, "a"));
+		inputs2.add(new Tuple2<Integer, String>(0, "b"));
+		inputs2.add(new Tuple2<Integer, String>(2, "a"));
+		inputs2.add(new Tuple2<Integer, String>(-1, "a"));
+		inputs2.add(new Tuple2<Integer, String>(-2, "a"));
+		inputs2.add(new Tuple2<Integer, String>(10, "a"));
+		inputs2.add(new Tuple2<Integer, String>(2, "b"));
+		inputs2.add(new Tuple2<Integer, String>(1, "a"));
+
+		List<Tuple2<Integer, String>> expected2 = new ArrayList<Tuple2<Integer, String>>();
+		expected2.add(new Tuple2<Integer, String>(-1, "a"));
+		expected2.add(new Tuple2<Integer, String>(-2, "a"));
+		expected2.add(new Tuple2<Integer, String>(0, "b"));
+
+		LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+		// Trigger on every 2nd element, but the first time after the 3rd
+		triggers.add(new CountTriggerPolicy<Tuple2<Integer, String>>(3));
+
+		LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+		// On every 2nd element, remove the oldest 2 elements, but the first
+		// time after the 3rd element
+		evictions.add(new TumblingEvictionPolicy<Tuple2<Integer, String>>());
+
+		LinkedList<TriggerPolicy<Tuple2<Integer, String>>> centralTriggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+
+		GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable2 = new GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+				new ReduceFunction<Tuple2<Integer, String>>() {
+					private static final long serialVersionUID = 1L;
+
+					@Override
+					public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+							Tuple2<Integer, String> value2) throws Exception {
+						if (value1.f0 <= value2.f0) {
+							return value1;
+						} else {
+							return value2;
+						}
+					}
+				}, keySelector, triggers, evictions, centralTriggers, null);
+
+		List<Tuple2<Integer, String>> result = MockContext.createAndExecute(invokable2, inputs2);
+
+		List<Tuple2<Integer, String>> actual2 = new LinkedList<Tuple2<Integer, String>>();
+		for (Tuple2<Integer, String> current : result) {
+			actual2.add(current);
+		}
+
+		assertEquals(new HashSet<Tuple2<Integer, String>>(expected2),
+				new HashSet<Tuple2<Integer, String>>(actual2));
+		assertEquals(expected2.size(), actual2.size());
+	}
+
+	/**
+	 * Test for active centralized trigger
+	 */
+	@Test
+	public void testGroupedWindowInvokableCentralActiveTrigger() {
+
+		List<Tuple2<Integer, String>> inputs = new ArrayList<Tuple2<Integer, String>>();
+		inputs.add(new Tuple2<Integer, String>(1, "a"));
+		inputs.add(new Tuple2<Integer, String>(1, "b"));
+		inputs.add(new Tuple2<Integer, String>(1, "c"));
+		inputs.add(new Tuple2<Integer, String>(2, "a"));
+		inputs.add(new Tuple2<Integer, String>(2, "b"));
+		inputs.add(new Tuple2<Integer, String>(2, "c"));
+		inputs.add(new Tuple2<Integer, String>(2, "b"));
+		inputs.add(new Tuple2<Integer, String>(2, "a"));
+		inputs.add(new Tuple2<Integer, String>(2, "c"));
+		inputs.add(new Tuple2<Integer, String>(3, "c"));
+		inputs.add(new Tuple2<Integer, String>(3, "a"));
+		inputs.add(new Tuple2<Integer, String>(3, "b"));
+		inputs.add(new Tuple2<Integer, String>(4, "a"));
+		inputs.add(new Tuple2<Integer, String>(4, "b"));
+		inputs.add(new Tuple2<Integer, String>(4, "c"));
+		inputs.add(new Tuple2<Integer, String>(5, "c"));
+		inputs.add(new Tuple2<Integer, String>(5, "a"));
+		inputs.add(new Tuple2<Integer, String>(5, "b"));
+		inputs.add(new Tuple2<Integer, String>(10, "b"));
+		inputs.add(new Tuple2<Integer, String>(10, "a"));
+		inputs.add(new Tuple2<Integer, String>(10, "c"));
+		inputs.add(new Tuple2<Integer, String>(11, "a"));
+		inputs.add(new Tuple2<Integer, String>(11, "a"));
+		inputs.add(new Tuple2<Integer, String>(11, "c"));
+		inputs.add(new Tuple2<Integer, String>(11, "c"));
+		inputs.add(new Tuple2<Integer, String>(11, "b"));
+		inputs.add(new Tuple2<Integer, String>(11, "b"));
+
+		// Expected result:
+		// For each group (a,b and c):
+		// 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+		// 12-12-5-10-32
+
+		List<Tuple2<Integer, String>> expected = new ArrayList<Tuple2<Integer, String>>();
+		expected.add(new Tuple2<Integer, String>(12, "a"));
+		expected.add(new Tuple2<Integer, String>(12, "b"));
+		expected.add(new Tuple2<Integer, String>(12, "c"));
+		expected.add(new Tuple2<Integer, String>(12, "a"));
+		expected.add(new Tuple2<Integer, String>(12, "b"));
+		expected.add(new Tuple2<Integer, String>(12, "c"));
+		expected.add(new Tuple2<Integer, String>(5, "a"));
+		expected.add(new Tuple2<Integer, String>(5, "b"));
+		expected.add(new Tuple2<Integer, String>(5, "c"));
+		expected.add(new Tuple2<Integer, String>(10, "a"));
+		expected.add(new Tuple2<Integer, String>(10, "b"));
+		expected.add(new Tuple2<Integer, String>(10, "c"));
+		expected.add(new Tuple2<Integer, String>(32, "a"));
+		expected.add(new Tuple2<Integer, String>(32, "b"));
+		expected.add(new Tuple2<Integer, String>(32, "c"));
+
+		Timestamp<Tuple2<Integer, String>> myTimeStamp = new Timestamp<Tuple2<Integer, String>>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public long getTimestamp(Tuple2<Integer, String> value) {
+				return value.f0;
+			}
+		};
+
+		TimestampWrapper<Tuple2<Integer, String>> myTimeStampWrapper = new TimestampWrapper<Tuple2<Integer, String>>(
+				myTimeStamp, 1);
+
+		ReduceFunction<Tuple2<Integer, String>> myReduceFunction = new ReduceFunction<Tuple2<Integer, String>>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1,
+					Tuple2<Integer, String> value2) throws Exception {
+				return new Tuple2<Integer, String>(value1.f0 + value2.f0, value1.f1);
+			}
+		};
+
+		LinkedList<TriggerPolicy<Tuple2<Integer, String>>> triggers = new LinkedList<TriggerPolicy<Tuple2<Integer, String>>>();
+		// Trigger every 2 time units but delay the first trigger by 2 (First
+		// trigger after 4, then every 2)
+		triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
+
+		LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>> evictions = new LinkedList<CloneableEvictionPolicy<Tuple2<Integer, String>>>();
+		// Always delete all elements older then 4
+		evictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
+
+		LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Tuple2<Integer, String>>>();
+
+		GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>> invokable = new GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+				myReduceFunction, keySelector, distributedTriggers, evictions, triggers, null);
+
+		ArrayList<Tuple2<Integer, String>> result = new ArrayList<Tuple2<Integer, String>>();
+		for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
+			result.add(t);
+		}
+
+		assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
+				new HashSet<Tuple2<Integer, String>>(result));
+		assertEquals(expected.size(), result.size());
+
+		// repeat the test with central eviction. The result should be the same.
+		triggers.clear();
+		triggers.add(new TimeTriggerPolicy<Tuple2<Integer, String>>(2L, myTimeStampWrapper, 2L));
+		evictions.clear();
+		LinkedList<EvictionPolicy<Tuple2<Integer, String>>> centralEvictions = new LinkedList<EvictionPolicy<Tuple2<Integer, String>>>();
+		centralEvictions.add(new TimeEvictionPolicy<Tuple2<Integer, String>>(4L, myTimeStampWrapper));
+
+		invokable = new GroupedStreamDiscretizer<Tuple2<Integer, String>, Tuple2<Integer, String>>(
+				myReduceFunction, keySelector, distributedTriggers, evictions, triggers,
+				centralEvictions);
+
+		result = new ArrayList<Tuple2<Integer, String>>();
+		for (Tuple2<Integer, String> t : MockContext.createAndExecute(invokable, inputs)) {
+			result.add(t);
+		}
+
+		assertEquals(new HashSet<Tuple2<Integer, String>>(expected),
+				new HashSet<Tuple2<Integer, String>>(result));
+		assertEquals(expected.size(), result.size());
+	}
+
+	/**
+	 * Test for multiple centralized trigger
+	 */
+	@Test
+	public void testGroupedWindowInvokableMultipleCentralTrigger() {
+		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+		triggers.add(new CountTriggerPolicy<Integer>(8));
+		triggers.add(new CountTriggerPolicy<Integer>(5));
+
+		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+		// The active wrapper causes eviction even on (fake) elements which
+		// triggered, but does not belong to the group.
+		evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+				new TumblingEvictionPolicy<Integer>()));
+
+		LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(2);
+		inputs.add(2);
+		inputs.add(2);
+		inputs.add(1);
+		// 1st Trigger: 2;6
+		inputs.add(2);
+		inputs.add(1);
+		inputs.add(2);
+		// 2nd Trigger: 1;4
+		inputs.add(2);
+		inputs.add(1);
+		// Final: 1,2
+
+		List<Integer> expected = new ArrayList<Integer>();
+		expected.add(2);
+		expected.add(6);
+		expected.add(4);
+		expected.add(1);
+		expected.add(2);
+		expected.add(1);
+
+		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer reduce(Integer value1, Integer value2) throws Exception {
+				return value1 + value2;
+			}
+		};
+
+		GroupedStreamDiscretizer<Integer, Integer> invokable = new GroupedStreamDiscretizer<Integer, Integer>(
+				myReduceFunction, new KeySelector<Integer, Integer>() {
+					private static final long serialVersionUID = 1L;
+
+					@Override
+					public Integer getKey(Integer value) {
+						return value;
+					}
+				}, distributedTriggers, evictions, triggers, null);
+
+		ArrayList<Integer> result = new ArrayList<Integer>();
+		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+			result.add(t);
+		}
+
+		assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+		assertEquals(expected.size(), result.size());
+	}
+
+	/**
+	 * Test for combination of centralized trigger and distributed trigger at
+	 * the same time
+	 */
+	@Test
+	public void testGroupedWindowInvokableCentralAndDistrTrigger() {
+		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+		triggers.add(new CountTriggerPolicy<Integer>(8));
+		triggers.add(new CountTriggerPolicy<Integer>(5));
+
+		LinkedList<CloneableEvictionPolicy<Integer>> evictions = new LinkedList<CloneableEvictionPolicy<Integer>>();
+		// The active wrapper causes eviction even on (fake) elements which
+		// triggered, but does not belong to the group.
+		evictions.add(new ActiveCloneableEvictionPolicyWrapper<Integer>(
+				new TumblingEvictionPolicy<Integer>()));
+
+		LinkedList<CloneableTriggerPolicy<Integer>> distributedTriggers = new LinkedList<CloneableTriggerPolicy<Integer>>();
+		distributedTriggers.add(new CountTriggerPolicy<Integer>(2));
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(2);
+		inputs.add(2);
+		// local on 2 => 4
+		inputs.add(2);
+		inputs.add(1);
+		// local on 1 => 2
+		// and 1st Central: 2;2
+		// SUMS up to 2;2
+		inputs.add(2);
+		// local on 2 => 2
+		inputs.add(1);
+		inputs.add(2);
+		// 2nd Central: 1;2
+		inputs.add(2);
+		inputs.add(1);
+		// Final: 1,2
+
+		List<Integer> expected = new ArrayList<Integer>();
+		expected.add(4);
+		expected.add(2);
+		expected.add(2);
+		expected.add(2);
+		expected.add(1);
+		expected.add(2);
+		expected.add(1);
+		expected.add(2);
+
+		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer reduce(Integer value1, Integer value2) throws Exception {
+				return value1 + value2;
+			}
+		};
+
+		GroupedStreamDiscretizer<Integer, Integer> invokable = new GroupedStreamDiscretizer<Integer, Integer>(
+				myReduceFunction, new KeySelector<Integer, Integer>() {
+					private static final long serialVersionUID = 1L;
+
+					@Override
+					public Integer getKey(Integer value) {
+						return value;
+					}
+				}, distributedTriggers, evictions, triggers, null);
+
+		ArrayList<Integer> result = new ArrayList<Integer>();
+		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+			result.add(t);
+		}
+
+		assertEquals(new HashSet<Integer>(expected), new HashSet<Integer>(result));
+		assertEquals(expected.size(), result.size());
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java
new file mode 100644
index 0000000..bba134a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/invokable/operator/windowing/WindowInvokableTest.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.invokable.operator.windowing;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Timestamp;
+import org.apache.flink.streaming.api.windowing.helper.TimestampWrapper;
+import org.apache.flink.streaming.api.windowing.policy.CountEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.CountTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.EvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeEvictionPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TimeTriggerPolicy;
+import org.apache.flink.streaming.api.windowing.policy.TriggerPolicy;
+import org.apache.flink.streaming.util.MockContext;
+import org.junit.Test;
+
+public class WindowInvokableTest {
+
+	/**
+	 * Test case equal to {@link WindowReduceInvokableTest}
+	 */
+	@Test
+	public void testWindowInvokableWithTimePolicy() {
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		inputs.add(1);
+		inputs.add(2);
+		inputs.add(2);
+		inputs.add(3);
+		inputs.add(4);
+		inputs.add(5);
+		inputs.add(10);
+		inputs.add(11);
+		inputs.add(11);
+		// 1,2,3,4-3,4,5,6-5,6,7,8-7,8,9,10-9,10,11
+		// 12-12-5-10-32
+
+		List<Integer> expected = new ArrayList<Integer>();
+		expected.add(12);
+		expected.add(12);
+		expected.add(5);
+		expected.add(10);
+		expected.add(32);
+
+		Timestamp<Integer> myTimeStamp = new Timestamp<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public long getTimestamp(Integer value) {
+				return value;
+			}
+		};
+
+		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer reduce(Integer value1, Integer value2) throws Exception {
+				return value1 + value2;
+			}
+		};
+
+		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+		// Trigger every 2 time units but delay the first trigger by 2 (First
+		// trigger after 4, then every 2)
+		triggers.add(new TimeTriggerPolicy<Integer>(2L, new TimestampWrapper<Integer>(myTimeStamp,
+				1), 2L));
+
+		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+		// Always delete all elements older then 4
+		evictions.add(new TimeEvictionPolicy<Integer>(4L, new TimestampWrapper<Integer>(
+				myTimeStamp, 1)));
+
+		WindowInvokable<Integer, Integer> invokable = new WindowReducer<Integer>(
+				myReduceFunction, triggers, evictions);
+
+		ArrayList<Integer> result = new ArrayList<Integer>();
+		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+			result.add(t);
+		}
+
+		assertEquals(expected, result);
+	}
+
+	/**
+	 * Test case equal to {@link BatchReduceTest}
+	 */
+	@Test
+	public void testWindowInvokableWithCountPolicy() {
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		for (Integer i = 1; i <= 10; i++) {
+			inputs.add(i);
+		}
+
+		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer reduce(Integer value1, Integer value2) throws Exception {
+				return value1 + value2;
+			}
+		};
+
+		/*
+		 * The following setup reassembles the batch size 3 and the slide size 2
+		 * of the BatchReduceInvokable.
+		 */
+		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+		// Trigger on every 2nd element, but the first time after the 3rd
+		triggers.add(new CountTriggerPolicy<Integer>(2, -1));
+
+		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+		// On every 2nd element, remove the oldest 2 elements, but the first
+		// time after the 3rd element
+		evictions.add(new CountEvictionPolicy<Integer>(2, 2, -1));
+
+		WindowInvokable<Integer, Integer> invokable = new WindowReducer<Integer>(
+				myReduceFunction, triggers, evictions);
+
+		List<Integer> expected = new ArrayList<Integer>();
+		expected.add(6);
+		expected.add(12);
+		expected.add(18);
+		expected.add(24);
+		expected.add(19);
+		List<Integer> result = new ArrayList<Integer>();
+		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+			result.add(t);
+		}
+		assertEquals(expected, result);
+
+		/*
+		 * Begin test part 2
+		 */
+
+		List<Integer> inputs2 = new ArrayList<Integer>();
+		inputs2.add(1);
+		inputs2.add(2);
+		inputs2.add(-5); // changed this value to make sure it is excluded from
+							// the result
+		inputs2.add(-3);
+		inputs2.add(-4);
+
+		myReduceFunction = new ReduceFunction<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer reduce(Integer value1, Integer value2) throws Exception {
+				if (value1 <= value2) {
+					return value1;
+				} else {
+					return value2;
+				}
+			};
+		};
+
+		/*
+		 * The following setup reassembles the batch size 2 and the slide size 3
+		 * of the BatchReduceInvokable.
+		 */
+		triggers = new LinkedList<TriggerPolicy<Integer>>();
+		// Trigger after every 3rd element, but the first time after the 2nd
+		triggers.add(new CountTriggerPolicy<Integer>(3, 1));
+
+		evictions = new LinkedList<EvictionPolicy<Integer>>();
+		// On every 3rd element, remove the oldest 3 elements, but the first
+		// time after on the 5th element
+		evictions.add(new CountEvictionPolicy<Integer>(3, 3, -1));
+
+		WindowInvokable<Integer, Integer> invokable2 = new WindowReducer<Integer>(
+				myReduceFunction, triggers, evictions);
+
+		List<Integer> expected2 = new ArrayList<Integer>();
+		expected2.add(1);
+		expected2.add(-4);
+
+		result = new ArrayList<Integer>();
+		for (Integer t : MockContext.createAndExecute(invokable2, inputs2)) {
+			result.add(t);
+		}
+
+		assertEquals(expected2, result);
+
+	}
+
+	@Test
+	public void testWindowInvokableWithMultiplePolicies() {
+		LinkedList<TriggerPolicy<Integer>> triggers = new LinkedList<TriggerPolicy<Integer>>();
+		triggers.add(new CountTriggerPolicy<Integer>(2));
+		triggers.add(new CountTriggerPolicy<Integer>(3));
+
+		LinkedList<EvictionPolicy<Integer>> evictions = new LinkedList<EvictionPolicy<Integer>>();
+		evictions.add(new CountEvictionPolicy<Integer>(2, 2));
+		evictions.add(new CountEvictionPolicy<Integer>(3, 3));
+
+		List<Integer> inputs = new ArrayList<Integer>();
+		for (Integer i = 1; i <= 10; i++) {
+			inputs.add(i);
+		}
+		/**
+		 * <code>
+		 * VAL: 1,2,3,4,5,6,7,8,9,10
+		 * TR1:   |   |   |   |   |
+		 * TR2:     |     |     |
+		 * EV1:   2   2   2   2   2
+		 * EV2:     3     3     3
+		 * </code>
+		 */
+
+		List<Integer> expected = new ArrayList<Integer>();
+		expected.add(3);
+		expected.add(3);
+		expected.add(4);
+		expected.add(11);
+		expected.add(15);
+		expected.add(9);
+		expected.add(10);
+
+		ReduceFunction<Integer> myReduceFunction = new ReduceFunction<Integer>() {
+			private static final long serialVersionUID = 1L;
+
+			@Override
+			public Integer reduce(Integer value1, Integer value2) throws Exception {
+				return value1 + value2;
+			}
+		};
+
+		WindowInvokable<Integer, Integer> invokable = new WindowReducer<Integer>(
+				myReduceFunction, triggers, evictions);
+
+		ArrayList<Integer> result = new ArrayList<Integer>();
+		for (Integer t : MockContext.createAndExecute(invokable, inputs)) {
+			result.add(t);
+		}
+
+		assertEquals(expected, result);
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
index 375c86d..a61adac 100755
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
@@ -63,7 +63,7 @@ public class IncrementalLearningSkeleton {
 		// build new model on every second of new data
 		DataStream<Double[]> model = env.addSource(new TrainingDataSource())
 				.window(Time.of(5000, TimeUnit.MILLISECONDS))
-				.reduceGroup(new PartialModelBuilder());
+				.mapWindow(new PartialModelBuilder()).flatten();
 
 		// use partial model for prediction
 		DataStream<Integer> prediction = env.addSource(new NewDataSource()).connect(model)

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
index e9b60f4..663541e 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/socket/SocketTextStreamWordCount.java
@@ -17,9 +17,13 @@
 
 package org.apache.flink.streaming.examples.socket;
 
+import java.util.concurrent.TimeUnit;
+
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.WindowedDataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.windowing.helper.Time;
 import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
 
 /**
@@ -51,29 +55,19 @@ import org.apache.flink.streaming.examples.wordcount.WordCount.Tokenizer;
 public class SocketTextStreamWordCount {
 	public static void main(String[] args) throws Exception {
 
-		if (!parseParameters(args)) {
-			return;
-		}
-
 		// set up the execution environment
-		final StreamExecutionEnvironment env = StreamExecutionEnvironment
-				.getExecutionEnvironment();
+		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
 		// get input data
-		DataStream<String> text = env.socketTextStream(hostName, port);
+		DataStream<String> text = env.socketTextStream("localhost", 9999);
 
-		DataStream<Tuple2<String, Integer>> counts =
+		WindowedDataStream<Tuple2<String, Integer>> counts =
 		// split up the lines in pairs (2-tuples) containing: (word,1)
 		text.flatMap(new Tokenizer())
 		// group by the tuple field "0" and sum up tuple field "1"
-				.groupBy(0)
-				.sum(1);
+				.window(Time.of(5, TimeUnit.SECONDS)).groupBy(0).sum(1);
 
-		if (fileOutput) {
-			counts.writeAsText(outputPath, 1);
-		} else {
-			counts.print();
-		}
+		counts.print();
 
 		// execute program
 		env.execute("WordCount from SocketTextStream Example");
@@ -100,7 +94,8 @@ public class SocketTextStreamWordCount {
 			hostName = args[0];
 			port = Integer.valueOf(args[1]);
 		} else {
-			System.err.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]");
+			System.err
+					.println("Usage: SocketTextStreamWordCount <hostname> <port> [<output path>]");
 			return false;
 		}
 		return true;

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
index 0f5d8eb..5e73fd6 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/java/org/apache/flink/streaming/examples/windowing/TopSpeedWindowingExample.java
@@ -60,7 +60,7 @@ public class TopSpeedWindowingExample {
 									Tuple4<Integer, Integer, Double, Long> newDataPoint) {
 								return newDataPoint.f2 - oldDataPoint.f2;
 							}
-						}, new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l))).maxBy(1);
+						}, new Tuple4<Integer, Integer, Double, Long>(0, 0, 0d, 0l))).local().maxBy(1).flatten();
 
 		topSpeeds.print();
 		env.execute("CarTopSpeedWindowingExample");

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
index a36a03b..6ea7be0 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/join/WindowJoin.scala
@@ -35,40 +35,31 @@ object WindowJoin {
   def main(args: Array[String]) {
 
     val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setDegreeOfParallelism(1)
 
-    //Create streams for names and ages by mapping the inputs to the corresponding objects
-    val names = env.fromCollection(nameStream).map(x => Name(x._1, x._2))
-    val ages = env.fromCollection(ageStream).map(x => Age(x._1, x._2))
-
-    //Join the two input streams by id on the last 2 seconds every second and create new 
-    //Person objects containing both name and age
-    val joined =
-      names.join(ages).onWindow(2, TimeUnit.SECONDS)
-                      .every(1, TimeUnit.SECONDS)
-                      .where("id")
-                      .equalTo("id") { (n, a) => Person(n.name, a.age) }
-
-    joined print
+    val split = env.generateSequence(1, 10).split(x => x % 3 toString)
+    
+    split.select("0").merge(split.select("1")).print
 
     env.execute("WindowJoin")
   }
 
-  def nameStream() : Stream[(Long,String)] = {
-    def nameMapper(names: Array[String])(x: Int) : (Long, String) =
-    {
-      if(x%100==0) Thread.sleep(1000)
-      (x, names(Random.nextInt(names.length)))
-    }
-    range(1,10000).map(nameMapper(Array("tom", "jerry", "alice", "bob", "john", "grace")))
+  def nameStream(): Stream[(Long, String)] = {
+    def nameMapper(names: Array[String])(x: Int): (Long, String) =
+      {
+        if (x % 100 == 0) Thread.sleep(1000)
+        (x, names(Random.nextInt(names.length)))
+      }
+    range(1, 10000).map(nameMapper(Array("tom", "jerry", "alice", "bob", "john", "grace")))
   }
 
-  def ageStream() : Stream[(Long,Int)] = {
-    def ageMapper(x: Int) : (Long, Int) =
-    {
-      if(x%100==0) Thread.sleep(1000)
-      (x, Random.nextInt(90))
-    }
-    range(1,10000).map(ageMapper)
+  def ageStream(): Stream[(Long, Int)] = {
+    def ageMapper(x: Int): (Long, Int) =
+      {
+        if (x % 100 == 0) Thread.sleep(1000)
+        (x, Random.nextInt(90))
+      }
+    range(1, 10000).map(ageMapper)
   }
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
index 714686e..c94901d 100644
--- a/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
+++ b/flink-staging/flink-streaming/flink-streaming-examples/src/main/scala/org/apache/flink/streaming/scala/examples/windowing/TopSpeedWindowing.scala
@@ -49,6 +49,7 @@ object TopSpeedWindowing {
       .window(Time.of(evictionSec, SECONDS))
       .every(Delta.of[CarEvent](triggerMeters,
           (oldSp,newSp) => newSp.distance-oldSp.distance, CarEvent(0,0,0,0)))
+      .local    
       .maxBy("speed")
 
     cars print

http://git-wip-us.apache.org/repos/asf/flink/blob/c560d76f/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
index 33bbc67..b22e527 100644
--- a/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
+++ b/flink-staging/flink-streaming/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/WindowedDataStream.scala
@@ -37,7 +37,7 @@ import org.apache.flink.streaming.api.windowing.helper.WindowingHelper
 import org.apache.flink.streaming.api.windowing.helper._
 import org.apache.flink.util.Collector
 
-class WindowedDataStream[T](javaStream: JavaWStream[T]) {
+class WindowedDataStream[T](javaStream: JavaWStream[WindowedDataStreamOld]) {
 
   /**
    * Defines the slide size (trigger frequency) for the windowed data stream.
@@ -87,6 +87,16 @@ class WindowedDataStream[T](javaStream: JavaWStream[T]) {
     }
     javaStream.groupBy(keyExtractor)
   }
+  
+  /**
+   * Sets the computations local meaning that the windowing and reduce or
+   * aggregation logic will be computed for each parallel instance of this
+   * operator
+   * 
+   */
+  def local(): WindowedDataStream[T]= {
+    javaStream.local
+  }
 
   /**
    * Applies a reduce transformation on the windowed data stream by reducing