You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2015/06/25 19:21:39 UTC

[06/12] flink git commit: [streaming] Allow multiple operator states + stateful function test rework

[streaming] Allow multiple operator states + stateful function test rework


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ef11e636
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ef11e636
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ef11e636

Branch: refs/heads/master
Commit: ef11e636154c343c6ba557060fb0df3ddcb3c77a
Parents: a7e2458
Author: Gyula Fora <gy...@apache.org>
Authored: Thu May 28 13:31:07 2015 +0200
Committer: Gyula Fora <gy...@apache.org>
Committed: Thu Jun 25 16:38:06 2015 +0200

----------------------------------------------------------------------
 .../api/common/functions/RuntimeContext.java    |  30 ++-
 .../util/AbstractRuntimeUDFContext.java         |   4 +-
 .../flink/api/common/state/OperatorState.java   |  14 +-
 .../runtime/state/PartitionedStateStore.java    |   4 +
 .../api/datastream/IterativeDataStream.java     |   2 -
 .../api/operators/AbstractStreamOperator.java   |   5 +
 .../operators/AbstractUdfStreamOperator.java    |  34 +--
 .../streaming/api/operators/StreamOperator.java |   2 +
 .../streaming/api/state/EagerStateStore.java    |  11 +-
 .../streaming/api/state/LazyStateStore.java     |  11 +-
 .../state/PartitionedStreamOperatorState.java   |   4 +-
 .../api/state/StreamOperatorState.java          |   9 +-
 .../runtime/tasks/OneInputStreamTask.java       |   1 +
 .../streaming/runtime/tasks/OutputHandler.java  |   6 +-
 .../streaming/runtime/tasks/StreamTask.java     |  29 ++-
 .../runtime/tasks/StreamingRuntimeContext.java  |  81 +++++--
 .../api/state/StatefulFunctionTest.java         | 211 -------------------
 .../api/state/StatefulOperatorTest.java         | 192 +++++++++++++++++
 .../runtime/tasks/SourceStreamTaskTest.java     |   2 +-
 .../flink/streaming/util/MockCoContext.java     |   2 +-
 .../flink/streaming/util/MockContext.java       |   2 +-
 .../StreamCheckpointingITCase.java              |   8 +-
 .../ProcessFailureStreamingRecoveryITCase.java  |   4 +-
 23 files changed, 369 insertions(+), 299 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
index a3b8f65..2c2ec53 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
@@ -166,10 +166,11 @@ public interface RuntimeContext {
 	// --------------------------------------------------------------------------------------------
 
 	/**
-	 * Returns the {@link OperatorState} of this operator instance, which can be
-	 * used to store and update user state in a fault tolerant fashion. The
-	 * state will be initialized by the provided default value, and the
-	 * {@link StateCheckpointer} will be used to draw the state snapshots.
+	 * Returns the {@link OperatorState} with the given name of the underlying
+	 * operator instance, which can be used to store and update user state in a
+	 * fault tolerant fashion. The state will be initialized by the provided
+	 * default value, and the {@link StateCheckpointer} will be used to draw the
+	 * state snapshots.
 	 * 
 	 * <p>
 	 * When storing a {@link Serializable} state the user can omit the
@@ -177,6 +178,9 @@ public interface RuntimeContext {
 	 * the snapshot.
 	 * </p>
 	 * 
+	 * @param name
+	 *            Identifier for the state allowing that more operator states
+	 *            can be used by the same operator.
 	 * @param defaultState
 	 *            Default value for the operator state. This will be returned
 	 *            the first time {@link OperatorState#getState()} (for every
@@ -185,26 +189,30 @@ public interface RuntimeContext {
 	 * @param checkpointer
 	 *            The {@link StateCheckpointer} that will be used to draw
 	 *            snapshots from the user state.
-	 * @return The {@link OperatorState} for this instance.
+	 * @return The {@link OperatorState} for the underlying operator.
 	 */
-	<S,C extends Serializable> OperatorState<S> getOperatorState(S defaultState, StateCheckpointer<S,C> checkpointer);
+	<S,C extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState, StateCheckpointer<S,C> checkpointer);
 
 	/**
-	 * Returns the {@link OperatorState} of this operator instance, which can be
-	 * used to store and update user state in a fault tolerant fashion. The
-	 * state will be initialized by the provided default value.
+	 * Returns the {@link OperatorState} with the given name of the underlying
+	 * operator instance, which can be used to store and update user state in a
+	 * fault tolerant fashion. The state will be initialized by the provided
+	 * default value.
 	 * 
 	 * <p>
 	 * When storing a non-{@link Serializable} state the user needs to specify a
 	 * {@link StateCheckpointer} for drawing snapshots.
 	 * </p>
 	 * 
+	 * @param name
+	 *            Identifier for the state allowing that more operator states can be
+	 *            used by the same operator.
 	 * @param defaultState
 	 *            Default value for the operator state. This will be returned
 	 *            the first time {@link OperatorState#getState()} (for every
 	 *            state partition) is called before
 	 *            {@link OperatorState#updateState(Object)}.
-	 * @return The {@link OperatorState} for this instance.
+	 * @return The {@link OperatorState} for the underlying operator.
 	 */
-	<S extends Serializable> OperatorState<S> getOperatorState(S defaultState);
+	<S extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState);
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
index 413565b..9f17bd1 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
@@ -174,12 +174,12 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
 	}
 	
 	@Override
-	public <S, C extends Serializable> OperatorState<S> getOperatorState(S defaultState, StateCheckpointer<S, C> checkpointer) {
+	public <S, C extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState, StateCheckpointer<S, C> checkpointer) {
 		throw new UnsupportedOperationException("Operator state is only accessible for streaming operators.");
 	}
 
 	@Override
-	public <S extends Serializable> OperatorState<S> getOperatorState(S defaultState) {
+	public <S extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState) {
 		throw new UnsupportedOperationException("Operator state is only accessible for streaming operators.");
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
index 5b3fa05..9155037 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
@@ -21,7 +21,7 @@ package org.apache.flink.api.common.state;
 import org.apache.flink.api.common.functions.MapFunction;
 
 /**
- * Base class for all streaming operator states. It can represent both
+ * Base interface for all streaming operator states. It can represent both
  * partitioned (when state partitioning is defined in the program) or
  * non-partitioned user states.
  * 
@@ -30,7 +30,7 @@ import org.apache.flink.api.common.functions.MapFunction;
  * transformation call the operator represents, for instance inside
  * {@link MapFunction#map()} and invalid in
  * {@link #open(org.apache.flink.configuration.Configuration)} or
- * {@link #close()}.
+ * {@link #close()} methods.
  * 
  * @param <T>
  *            Type of the operator state
@@ -42,13 +42,7 @@ public interface OperatorState<T> {
 	 * partitioned the returned state is the same for all inputs. If state
 	 * partitioning is applied the state returned depends on the current
 	 * operator input, as the operator maintains an independent state for each
-	 * partitions.
-	 * 
-	 * <p>
-	 * {@link #getState()} returns <code>null</code> if there is no state stored
-	 * in the operator. This is the expected behaviour before initializing the
-	 * state with {@link #updateState(T)}.
-	 * </p>
+	 * partition.
 	 * 
 	 * @return The operator state corresponding to the current input.
 	 */
@@ -60,7 +54,7 @@ public interface OperatorState<T> {
 	 * partition) the returned state will represent the updated value.
 	 * 
 	 * @param state
-	 *            The updated state.
+	 *            The new state.
 	 */
 	void updateState(T state);
 	

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
index 8b73edf..6353eda 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
@@ -21,6 +21,8 @@ package org.apache.flink.runtime.state;
 import java.io.Serializable;
 import java.util.Map;
 
+import org.apache.flink.api.common.state.StateCheckpointer;
+
 /**
  * Interface for storing and accessing partitioned state. The interface is
  * designed in a way that allows implementations for lazily state access.
@@ -43,5 +45,7 @@ public interface PartitionedStateStore<S, C extends Serializable> {
 	void restoreStates(Map<Serializable, StateHandle<C>> snapshots) throws Exception;
 
 	boolean containsKey(Serializable key);
+	
+	void setCheckPointer(StateCheckpointer<S, C> checkpointer);
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index 2d70d49..6a48b6a 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -17,8 +17,6 @@
 
 package org.apache.flink.streaming.api.datastream;
 
-import org.apache.flink.streaming.api.collector.selector.OutputSelector;
-
 /**
  * The iterative data stream represents the start of an iteration in a
  * {@link DataStream}.

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 40f61d9..cb4bcb0 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -79,4 +79,9 @@ public abstract class AbstractStreamOperator<OUT> implements StreamOperator<OUT>
 	public void disableInputCopy() {
 		this.inputCopyDisabled = true;
 	}
+	
+	@Override
+	public StreamingRuntimeContext getRuntimeContext(){
+		return runtimeContext;
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
index cbcbcee..da0fdb7 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
@@ -19,7 +19,9 @@
 package org.apache.flink.streaming.api.operators;
 
 import java.io.Serializable;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.flink.api.common.functions.Function;
 import org.apache.flink.api.common.functions.util.FunctionUtils;
@@ -67,24 +69,32 @@ public abstract class AbstractUdfStreamOperator<OUT, F extends Function & Serial
 		FunctionUtils.closeFunction(userFunction);
 	}
 
-	@SuppressWarnings("unchecked")
+	@SuppressWarnings({ "unchecked", "rawtypes" })
 	public void restoreInitialState(Serializable state) throws Exception {
 
-		Map<Serializable, StateHandle<Serializable>> snapshots = (Map<Serializable, StateHandle<Serializable>>) state;
+		Map<String, Map<Serializable, StateHandle<Serializable>>> snapshots = (Map<String, Map<Serializable, StateHandle<Serializable>>>) state;
 
-		StreamOperatorState<?, Serializable> operatorState = (StreamOperatorState<?, Serializable>) runtimeContext
-				.getOperatorState();
-
-		operatorState.restoreState(snapshots);
+		Map<String, StreamOperatorState> operatorStates = runtimeContext.getOperatorStates();
+		
+		for (Entry<String, Map<Serializable, StateHandle<Serializable>>> snapshot : snapshots.entrySet()) {
+			StreamOperatorState restoredState = runtimeContext.createRawState();
+			restoredState.restoreState(snapshot.getValue());
+			operatorStates.put(snapshot.getKey(), restoredState);
+		}
 
 	}
 
-	public Serializable getStateSnapshotFromFunction(long checkpointId, long timestamp)
-			throws Exception {
-		
-		StreamOperatorState<?,?> operatorState = (StreamOperatorState<?,?>) runtimeContext.getOperatorState();
-		
-		return (Serializable) operatorState.snapshotState(checkpointId, timestamp); 
+	@SuppressWarnings({ "rawtypes", "unchecked" })
+	public Serializable getStateSnapshotFromFunction(long checkpointId, long timestamp) throws Exception {
+
+		Map<String, StreamOperatorState> operatorStates = runtimeContext.getOperatorStates();
+		Map<String, Map<Serializable, StateHandle<Serializable>>> snapshots = new HashMap<String, Map<Serializable, StateHandle<Serializable>>>();
+
+		for (Entry<String, StreamOperatorState> state : operatorStates.entrySet()) {
+			snapshots.put(state.getKey(), state.getValue().snapshotState(checkpointId, timestamp));
+		}
+
+		return (Serializable) snapshots;
 	}
 
 	public void confirmCheckpointCompleted(long checkpointId, long timestamp,

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index aebff5c..75cea5f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -48,6 +48,8 @@ public interface StreamOperator<OUT> extends Serializable {
 	 * This method is called after no more elements for can arrive for processing.
 	 */
 	public void close() throws Exception;
+	
+	public StreamingRuntimeContext getRuntimeContext();
 
 	/**
 	 * An operator can return true here to disable copying of its input elements. This overrides

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
index 4ac01a5..b374fe4 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
@@ -30,10 +30,10 @@ import org.apache.flink.runtime.state.StateHandleProvider;
 
 public class EagerStateStore<S, C extends Serializable> implements PartitionedStateStore<S, C> {
 
-	protected StateCheckpointer<S, C> checkpointer;
-	protected StateHandleProvider<C> provider;
+	private StateCheckpointer<S, C> checkpointer;
+	private final StateHandleProvider<C> provider;
 
-	Map<Serializable, S> fetchedState;
+	private Map<Serializable, S> fetchedState;
 
 	public EagerStateStore(StateCheckpointer<S, C> checkpointer, StateHandleProvider<C> provider) {
 		this.checkpointer = checkpointer;
@@ -83,4 +83,9 @@ public class EagerStateStore<S, C extends Serializable> implements PartitionedSt
 		return fetchedState.containsKey(key);
 	}
 
+	@Override
+	public void setCheckPointer(StateCheckpointer<S, C> checkpointer) {
+		this.checkpointer = checkpointer;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
index 9872a0c..14484ea 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
@@ -47,10 +47,10 @@ import org.apache.flink.runtime.state.StateHandleProvider;
 public class LazyStateStore<S, C extends Serializable> implements PartitionedStateStore<S, C> {
 
 	protected StateCheckpointer<S, C> checkpointer;
-	protected StateHandleProvider<C> provider;
+	protected final StateHandleProvider<C> provider;
 
-	Map<Serializable, StateHandle<C>> unfetchedState;
-	Map<Serializable, S> fetchedState;
+	private final Map<Serializable, StateHandle<C>> unfetchedState;
+	private final Map<Serializable, S> fetchedState;
 
 	public LazyStateStore(StateCheckpointer<S, C> checkpointer, StateHandleProvider<C> provider) {
 		this.checkpointer = checkpointer;
@@ -114,4 +114,9 @@ public class LazyStateStore<S, C extends Serializable> implements PartitionedSta
 		return fetchedState.containsKey(key) || unfetchedState.containsKey(key);
 	}
 
+	@Override
+	public void setCheckPointer(StateCheckpointer<S, C> checkpointer) {
+		this.checkpointer = checkpointer;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
index 26b2a88..4f0200f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
@@ -45,9 +45,9 @@ public class PartitionedStreamOperatorState<IN, S, C extends Serializable> exten
 		StreamOperatorState<S, C> {
 
 	// KeySelector for getting the state partition key for each input
-	private KeySelector<IN, Serializable> keySelector;
+	private final KeySelector<IN, Serializable> keySelector;
 
-	private PartitionedStateStore<S, C> stateStore;
+	private final PartitionedStateStore<S, C> stateStore;
 	
 	private S defaultState;
 

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
index 90a3726..bab3421 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
@@ -44,7 +44,7 @@ public class StreamOperatorState<S, C extends Serializable> implements OperatorS
 
 	private S state;
 	private StateCheckpointer<S, C> checkpointer;
-	private StateHandleProvider<C> provider;
+	private final StateHandleProvider<C> provider;
 
 	public StreamOperatorState(StateCheckpointer<S, C> checkpointer, StateHandleProvider<C> provider) {
 		this.checkpointer = checkpointer;
@@ -66,8 +66,11 @@ public class StreamOperatorState<S, C extends Serializable> implements OperatorS
 		this.state = state;
 	}
 	
-	public void setDefaultState(S defaultState){
-		updateState(defaultState);
+	public void setDefaultState(S defaultState) {
+		// reconsider this as it might cause issues when setting the state to null
+		if (getState() == null) {
+			updateState(defaultState);
+		}
 	}
 
 	public StateCheckpointer<S, C> getCheckpointer() {

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
index 6750b52..87042ba 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
@@ -99,6 +99,7 @@ public class OneInputStreamTask<IN, OUT> extends StreamTask<OUT, OneInputStreamO
 
 			StreamRecord<IN> nextRecord;
 			while (isRunning && (nextRecord = readNext()) != null) {
+				headContext.setNextInput(nextRecord);
 				streamOperator.processElement(nextRecord.getObject());
 			}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
index effce6c..a7755f6 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
@@ -226,9 +226,10 @@ public class OutputHandler<OUT> {
 	}
 
 	private static class OperatorCollector<T> implements Output<T> {
-		protected OneInputStreamOperator operator;
 
-		public OperatorCollector(OneInputStreamOperator<?, T> operator) {
+		protected OneInputStreamOperator<Object, T> operator;
+
+		public OperatorCollector(OneInputStreamOperator<Object, T> operator) {
 			this.operator = operator;
 		}
 
@@ -236,6 +237,7 @@ public class OutputHandler<OUT> {
 		@SuppressWarnings("unchecked")
 		public void collect(T record) {
 			try {
+				operator.getRuntimeContext().setNextInput(record);
 				operator.processElement(record);
 			} catch (Exception e) {
 				if (LOG.isErrorEnabled()) {

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index e7f4d9c..4ae72d2 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -42,8 +42,6 @@ import org.apache.flink.streaming.api.graph.StreamConfig;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
 import org.apache.flink.streaming.api.operators.StatefulStreamOperator;
 import org.apache.flink.streaming.api.operators.StreamOperator;
-import org.apache.flink.streaming.api.state.PartitionedStreamOperatorState;
-import org.apache.flink.streaming.api.state.StreamOperatorState;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -66,6 +64,8 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 	protected volatile boolean isRunning = false;
 
 	protected List<StreamingRuntimeContext> contexts;
+	
+	protected StreamingRuntimeContext headContext;
 
 	protected ClassLoader userClassLoader;
 	
@@ -93,7 +93,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 			// IterationHead and IterationTail don't have an Operator...
 
 			//Create context of the head operator
-			StreamingRuntimeContext headContext = createRuntimeContext(configuration);
+			headContext = createRuntimeContext(configuration);
 			this.contexts.add(headContext);
 			streamOperator.setup(outputHandler.getOutput(), headContext);
 		}
@@ -105,19 +105,14 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 		return getEnvironment().getTaskName();
 	}
 
-	@SuppressWarnings({ "rawtypes", "unchecked" })
 	public StreamingRuntimeContext createRuntimeContext(StreamConfig conf) {
 		Environment env = getEnvironment();
 		String operatorName = conf.getStreamOperator(userClassLoader).getClass().getSimpleName();
 		
 		KeySelector<?,Serializable> statePartitioner = conf.getStatePartitioner(userClassLoader);
 		
-		StreamOperatorState state = statePartitioner == null ? new StreamOperatorState(
-				getStateHandleProvider()) : new PartitionedStreamOperatorState(
-				getStateHandleProvider(), statePartitioner);
-		
 		return new StreamingRuntimeContext(operatorName, env, getUserCodeClassLoader(),
-				getExecutionConfig(), state);
+				getExecutionConfig(), statePartitioner, getStateHandleProvider());
 	}
 	
 	private StateHandleProvider<Serializable> getStateHandleProvider() {
@@ -216,16 +211,16 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 			Serializable headState = chainedStates.get(0);
 			if (headState != null) {
 				if (streamOperator instanceof StatefulStreamOperator) {
-					((StatefulStreamOperator) streamOperator).restoreInitialState(headState);
+					((StatefulStreamOperator<?>) streamOperator).restoreInitialState(headState);
 				}
 			}
 
 			for (int i = 1; i < chainedStates.size(); i++) {
 				Serializable chainedState = chainedStates.get(i);
 				if (chainedState != null) {
-					StreamOperator chainedOperator = outputHandler.getChainedOperators().get(i - 1);
+					StreamOperator<?> chainedOperator = outputHandler.getChainedOperators().get(i - 1);
 					if (chainedOperator instanceof StatefulStreamOperator) {
-						((StatefulStreamOperator) chainedOperator).restoreInitialState(chainedState);
+						((StatefulStreamOperator<?>) chainedOperator).restoreInitialState(chainedState);
 					}
 
 				}
@@ -233,7 +228,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 
 		} else {
 			if (streamOperator instanceof StatefulStreamOperator) {
-				((StatefulStreamOperator) streamOperator).restoreInitialState(state);
+				((StatefulStreamOperator<?>) streamOperator).restoreInitialState(state);
 			}
 
 		}
@@ -254,7 +249,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 						Serializable userState = null;
 
 						if (streamOperator instanceof StatefulStreamOperator) {
-							userState = ((StatefulStreamOperator) streamOperator).getStateSnapshotFromFunction(checkpointId, timestamp);
+							userState = ((StatefulStreamOperator<?>) streamOperator).getStateSnapshotFromFunction(checkpointId, timestamp);
 						}
 
 
@@ -266,7 +261,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 
 							for (OneInputStreamOperator<?, ?> chainedOperator : outputHandler.getChainedOperators()) {
 								if (chainedOperator instanceof StatefulStreamOperator) {
-									chainedStates.add(((StatefulStreamOperator) chainedOperator).getStateSnapshotFromFunction(checkpointId, timestamp));
+									chainedStates.add(((StatefulStreamOperator<?>) chainedOperator).getStateSnapshotFromFunction(checkpointId, timestamp));
 								}
 							}
 
@@ -306,13 +301,13 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
 		// we do nothing here so far. this should call commit on the source function, for example
 		synchronized (checkpointLock) {
 			if (streamOperator instanceof StatefulStreamOperator) {
-				((StatefulStreamOperator) streamOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
+				((StatefulStreamOperator<?>) streamOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
 			}
 
 			if (hasChainedOperators) {
 				for (OneInputStreamOperator<?, ?> chainedOperator : outputHandler.getChainedOperators()) {
 					if (chainedOperator instanceof StatefulStreamOperator) {
-						((StatefulStreamOperator) chainedOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
+						((StatefulStreamOperator<?>) chainedOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
 					}
 				}
 			}

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
index 5fd158c..77639a0 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
@@ -19,35 +19,44 @@
 package org.apache.flink.streaming.runtime.tasks;
 
 import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
 import org.apache.flink.api.common.state.OperatorState;
 import org.apache.flink.api.common.state.StateCheckpointer;
+import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
 import org.apache.flink.runtime.operators.util.TaskConfig;
+import org.apache.flink.runtime.state.StateHandleProvider;
+import org.apache.flink.streaming.api.state.PartitionedStreamOperatorState;
 import org.apache.flink.streaming.api.state.StreamOperatorState;
 
 /**
  * Implementation of the {@link RuntimeContext}, created by runtime stream UDF
  * operators.
  */
+@SuppressWarnings("rawtypes")
 public class StreamingRuntimeContext extends RuntimeUDFContext {
 
 	private final Environment env;
-	@SuppressWarnings("rawtypes")
-	private StreamOperatorState state;
-
+	private final Map<String, StreamOperatorState> states;
+	private final KeySelector<?, ?> statePartitioner;
+	private final StateHandleProvider<?> provider;
 
 	public StreamingRuntimeContext(String name, Environment env, ClassLoader userCodeClassLoader,
-			ExecutionConfig executionConfig, StreamOperatorState<?, ?> state) {
+			ExecutionConfig executionConfig, KeySelector<?, ?> statePartitioner,
+			StateHandleProvider<?> provider) {
 		super(name, env.getNumberOfSubtasks(), env.getIndexInSubtaskGroup(), userCodeClassLoader,
 				executionConfig, env.getDistributedCacheEntries());
 		this.env = env;
-		this.state = state;
+		this.statePartitioner = statePartitioner;
+		this.states = new HashMap<String, StreamOperatorState>();
+		this.provider = provider;
 	}
 
 	/**
@@ -68,25 +77,73 @@ public class StreamingRuntimeContext extends RuntimeUDFContext {
 	public Configuration getTaskStubParameters() {
 		return new TaskConfig(env.getTaskConfiguration()).getStubParameters();
 	}
-	
+
 	@SuppressWarnings("unchecked")
 	@Override
-	public <S, C extends Serializable> OperatorState<S> getOperatorState(S defaultState,
-			StateCheckpointer<S, C> checkpointer) {
+	public <S, C extends Serializable> OperatorState<S> getOperatorState(String name,
+			S defaultState, StateCheckpointer<S, C> checkpointer) {
+		StreamOperatorState state;
+		if (states.containsKey(name)) {
+			state = states.get(name);
+		} else {
+			state = createRawState();
+			states.put(name, state);
+		}
+		state.setDefaultState(defaultState);
 		state.setCheckpointer(checkpointer);
+
 		return (OperatorState<S>) state;
 	}
-	
+
 	@SuppressWarnings("unchecked")
 	@Override
-	public <S extends Serializable> OperatorState<S> getOperatorState(S defaultState) {
+	public <S extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState) {
+		StreamOperatorState state;
+		if (states.containsKey(name)) {
+			state = states.get(name);
+		} else {
+			state = createRawState();
+			states.put(name, state);
+		}
 		state.setDefaultState(defaultState);
+
 		return (OperatorState<S>) state;
 	}
+
+	/**
+	 * Creates an empty state depending on the partitioning state.
+	 * 
+	 * @return An empty operator state.
+	 */
+	@SuppressWarnings("unchecked")
+	public StreamOperatorState createRawState() {
+		if (statePartitioner == null) {
+			return new StreamOperatorState(provider);
+		} else {
+			return new PartitionedStreamOperatorState(provider, statePartitioner);
+		}
+	}
 	
+	/**
+	 * Provides access to the all the states contained in the context
+	 * 
+	 * @return All the states for the underlying operator.
+	 */
+	public Map<String, StreamOperatorState> getOperatorStates() {
+		return states;
+	}
+
+	/**
+	 * Sets the next input of the underlying operators, used to access partitioned states.
+	 * @param nextRecord Next input of the operator.
+	 */
 	@SuppressWarnings("unchecked")
-	public <S extends Serializable> OperatorState<S> getOperatorState() {
-		return (OperatorState<S>) state;
+	public void setNextInput(Object nextRecord) {
+		if (statePartitioner != null) {
+			for (StreamOperatorState state : states.values()) {
+				((PartitionedStreamOperatorState) state).setCurrentInput(nextRecord);
+			}
+		}
 	}
 
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java
deleted file mode 100644
index f4c1a89..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.state;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.DoubleCounter;
-import org.apache.flink.api.common.accumulators.Histogram;
-import org.apache.flink.api.common.accumulators.IntCounter;
-import org.apache.flink.api.common.accumulators.LongCounter;
-import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.state.OperatorState;
-import org.apache.flink.api.common.state.StateCheckpointer;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.state.LocalStateHandle.LocalStateHandleProvider;
-import org.apache.flink.runtime.state.StateHandle;
-import org.apache.flink.shaded.com.google.common.collect.ImmutableMap;
-import org.apache.flink.util.InstantiationUtil;
-import org.junit.Test;
-
-public class StatefulFunctionTest {
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void simpleStateTest() throws Exception {
-
-		StatefulMapper mapper = new StatefulMapper();
-		MockContext context = new MockContext(false, mapper);
-		mapper.setRuntimeContext(context);
-		mapper.open(null);
-
-		assertEquals(Arrays.asList("1", "2", "3", "4", "5"),
-				applyOnSequence(mapper, 1, 5, context.state));
-		assertEquals((Integer) 5, context.state.getState());
-
-		byte[] serializedState = InstantiationUtil.serializeObject(context.state
-				.snapshotState(1, 1));
-
-		StatefulMapper restoredMapper = new StatefulMapper();
-		MockContext restoredContext = new MockContext(false, restoredMapper);
-		restoredMapper.setRuntimeContext(context);
-		restoredMapper.open(null);
-
-		assertEquals(null, restoredContext.state.getState());
-
-		Map<Serializable, StateHandle<Integer>> deserializedState = (Map<Serializable, StateHandle<Integer>>) InstantiationUtil
-				.deserializeObject(serializedState, Thread.currentThread().getContextClassLoader());
-
-		restoredContext.state.restoreState(deserializedState);
-
-		assertEquals((Integer) 5, restoredContext.state.getState());
-
-	}
-
-	@SuppressWarnings("unchecked")
-	@Test
-	public void partitionedStateTest() throws Exception {
-		StatefulMapper mapper = new StatefulMapper();
-		MockContext context = new MockContext(true, mapper);
-		mapper.setRuntimeContext(context);
-		mapper.open(null);
-
-		assertEquals(Arrays.asList("1", "2", "3", "4", "5"),
-				applyOnSequence(mapper, 1, 5, context.state));
-		assertEquals(ImmutableMap.of(0, 2, 1, 3), context.state.getPartitionedState());
-
-		byte[] serializedState = InstantiationUtil.serializeObject(context.state
-				.snapshotState(1, 1));
-
-		StatefulMapper restoredMapper = new StatefulMapper();
-		MockContext restoredContext = new MockContext(true, restoredMapper);
-		restoredMapper.setRuntimeContext(context);
-		restoredMapper.open(null);
-
-		assertEquals(null, restoredContext.state.getState());
-
-		Map<Serializable, StateHandle<Integer>> deserializedState = (Map<Serializable, StateHandle<Integer>>) InstantiationUtil
-				.deserializeObject(serializedState, Thread.currentThread().getContextClassLoader());
-
-		restoredContext.state.restoreState(deserializedState);
-
-		assertEquals(ImmutableMap.of(0, 2, 1, 3), restoredContext.state.getPartitionedState());
-
-	}
-
-	@SuppressWarnings({ "rawtypes", "unchecked" })
-	private <T> List<T> applyOnSequence(MapFunction<Integer, T> mapper, int from, int to,
-			StreamOperatorState state) throws Exception {
-		List<T> output = new ArrayList<T>();
-		for (int i = from; i <= to; i++) {
-			if (state instanceof PartitionedStreamOperatorState) {
-				((PartitionedStreamOperatorState) state).setCurrentInput(i);
-			}
-			output.add(mapper.map(i));
-		}
-		return output;
-	}
-
-	public static class ModKey implements KeySelector<Integer, Serializable> {
-
-		private static final long serialVersionUID = 4193026742083046736L;
-
-		int base;
-
-		public ModKey(int base) {
-			this.base = base;
-		}
-
-		@Override
-		public Integer getKey(Integer value) throws Exception {
-			return value % base;
-		}
-
-	}
-
-	public static class StatefulMapper extends RichMapFunction<Integer, String> {
-
-		private static final long serialVersionUID = -9007873655253339356L;
-		OperatorState<Integer> opState;
-
-		@Override
-		public String map(Integer value) throws Exception {
-			opState.updateState(opState.getState() + 1);
-			return value.toString();
-		}
-
-		@Override
-		public void open(Configuration conf) {
-			opState = getRuntimeContext().getOperatorState(0);
-		}
-	}
-
-	public static class MockContext implements RuntimeContext {
-
-		StreamOperatorState<Integer, Integer> state;
-
-		public MockContext(boolean isPartitionedState, StatefulMapper mapper) {
-			if (isPartitionedState) {
-				this.state = new PartitionedStreamOperatorState<Integer, Integer, Integer>(
-						new LocalStateHandleProvider<Integer>(), new ModKey(2));
-			} else {
-				this.state = new StreamOperatorState<Integer, Integer>(
-						new LocalStateHandleProvider<Integer>());
-			}
-		}
-		
-		public String getTaskName() {return null;}
-		public int getNumberOfParallelSubtasks() {return 0;}
-		public int getIndexOfThisSubtask() {return 0;}
-		public ExecutionConfig getExecutionConfig() {return null;}
-		public ClassLoader getUserCodeClassLoader() {return null;}
-		public <V, A extends Serializable> void addAccumulator(String name, Accumulator<V, A> accumulator) {}
-		public <V, A extends Serializable> Accumulator<V, A> getAccumulator(String name) {return null;}
-		public HashMap<String, Accumulator<?, ?>> getAllAccumulators() {return null;}	
-		public IntCounter getIntCounter(String name) {return null;}	
-		public LongCounter getLongCounter(String name) {return null;}
-		public DoubleCounter getDoubleCounter(String name) {return null;}
-		public Histogram getHistogram(String name) {return null;}		
-		public <RT> List<RT> getBroadcastVariable(String name) {return null;}
-		public <T, C> C getBroadcastVariableWithInitializer(String name,
-				BroadcastVariableInitializer<T, C> initializer) {return null;}
-		public DistributedCache getDistributedCache() {return null;}
-
-		@SuppressWarnings("unchecked")
-		@Override
-		public <S, C extends Serializable> OperatorState<S> getOperatorState(S defaultState,
-				StateCheckpointer<S, C> checkpointer) {
-			state.setDefaultState((Integer) defaultState);
-			return (OperatorState<S>) state;
-		}
-
-		@SuppressWarnings("unchecked")
-		@Override
-		public <S extends Serializable> OperatorState<S> getOperatorState(S defaultState) {
-			state.setDefaultState((Integer) defaultState);
-			return (OperatorState<S>) state;
-		}
-
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java
new file mode 100644
index 0000000..99015c9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.state;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.state.OperatorState;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
+import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
+import org.apache.flink.runtime.state.LocalStateHandle.LocalStateHandleProvider;
+import org.apache.flink.shaded.com.google.common.collect.ImmutableMap;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.operators.StreamMap;
+import org.apache.flink.streaming.runtime.tasks.StreamingRuntimeContext;
+import org.apache.flink.util.InstantiationUtil;
+import org.junit.Test;
+
+/**
+ * Test the functionality supported by stateful user functions for both
+ * partitioned and non-partitioned user states. This test mimics the runtime
+ * behavior of stateful stream operators.
+ */
+public class StatefulOperatorTest {
+
+	@Test
+	public void simpleStateTest() throws Exception {
+
+		List<String> out = new ArrayList<String>();
+
+		StreamMap<Integer, String> map = createOperatorWithContext(out, null, null);
+		StreamingRuntimeContext context = map.getRuntimeContext();
+
+		processInputs(map, Arrays.asList(1, 2, 3, 4, 5));
+
+		assertEquals(Arrays.asList("1", "2", "3", "4", "5"), out);
+		assertEquals((Integer) 5, context.getOperatorState("counter", 0).getState());
+		assertEquals("12345", context.getOperatorState("concat", "").getState());
+
+		byte[] serializedState = InstantiationUtil.serializeObject(map.getStateSnapshotFromFunction(1, 1));
+
+		StreamMap<Integer, String> restoredMap = createOperatorWithContext(out, null, serializedState);
+		StreamingRuntimeContext restoredContext = restoredMap.getRuntimeContext();
+
+		assertEquals((Integer) 5, restoredContext.getOperatorState("counter", 0).getState());
+		assertEquals("12345", restoredContext.getOperatorState("concat", "").getState());
+		out.clear();
+
+		processInputs(restoredMap, Arrays.asList(7, 8));
+
+		assertEquals(Arrays.asList("7", "8"), out);
+		assertEquals((Integer) 7, restoredContext.getOperatorState("counter", 0).getState());
+		assertEquals("1234578", restoredContext.getOperatorState("concat", "").getState());
+
+	}
+
+	@Test
+	public void partitionedStateTest() throws Exception {
+		List<String> out = new ArrayList<String>();
+
+		StreamMap<Integer, String> map = createOperatorWithContext(out, new ModKey(2), null);
+		StreamingRuntimeContext context = map.getRuntimeContext();
+
+		processInputs(map, Arrays.asList(1, 2, 3, 4, 5));
+
+		assertEquals(Arrays.asList("1", "2", "3", "4", "5"), out);
+		assertEquals(ImmutableMap.of(0, 2, 1, 3), context.getOperatorStates().get("counter").getPartitionedState());
+		assertEquals(ImmutableMap.of(0, "24", 1, "135"), context.getOperatorStates().get("concat")
+				.getPartitionedState());
+
+		byte[] serializedState = InstantiationUtil.serializeObject(map.getStateSnapshotFromFunction(1, 1));
+
+		StreamMap<Integer, String> restoredMap = createOperatorWithContext(out, new ModKey(2), serializedState);
+		StreamingRuntimeContext restoredContext = restoredMap.getRuntimeContext();
+
+		assertEquals(ImmutableMap.of(0, 2, 1, 3), restoredContext.getOperatorStates().get("counter")
+				.getPartitionedState());
+		assertEquals(ImmutableMap.of(0, "24", 1, "135"), restoredContext.getOperatorStates().get("concat")
+				.getPartitionedState());
+		out.clear();
+
+		processInputs(restoredMap, Arrays.asList(7, 8));
+
+		assertEquals(Arrays.asList("7", "8"), out);
+		assertEquals(ImmutableMap.of(0, 3, 1, 4), restoredContext.getOperatorStates().get("counter")
+				.getPartitionedState());
+		assertEquals(ImmutableMap.of(0, "248", 1, "1357"), restoredContext.getOperatorStates().get("concat")
+				.getPartitionedState());
+
+	}
+
+	private void processInputs(StreamMap<Integer, ?> map, List<Integer> input) throws Exception {
+		for (Integer i : input) {
+			map.getRuntimeContext().setNextInput(i);
+			map.processElement(i);
+		}
+	}
+
+	private StreamMap<Integer, String> createOperatorWithContext(List<String> output,
+			KeySelector<Integer, Serializable> partitioner, byte[] serializedState) throws Exception {
+		final List<String> outputList = output;
+
+		StreamingRuntimeContext context = new StreamingRuntimeContext("MockTask", new MockEnvironment(3 * 1024 * 1024,
+				new MockInputSplitProvider(), 1024), null, new ExecutionConfig(), partitioner,
+				new LocalStateHandleProvider<Serializable>());
+
+		StreamMap<Integer, String> op = new StreamMap<Integer, String>(new StatefulMapper());
+
+		op.setup(new Output<String>() {
+
+			@Override
+			public void collect(String record) {
+				outputList.add(record);
+			}
+
+			@Override
+			public void close() {
+			}
+		}, context);
+
+		if (serializedState != null) {
+			op.restoreInitialState((Serializable) InstantiationUtil.deserializeObject(serializedState, Thread
+					.currentThread().getContextClassLoader()));
+		}
+
+		op.open(null);
+
+		return op;
+	}
+
+	public static class StatefulMapper extends RichMapFunction<Integer, String> {
+
+		private static final long serialVersionUID = -9007873655253339356L;
+		OperatorState<Integer> counter;
+		OperatorState<String> concat;
+
+		@Override
+		public String map(Integer value) throws Exception {
+			counter.updateState(counter.getState() + 1);
+			concat.updateState(concat.getState() + value.toString());
+			return value.toString();
+		}
+
+		@Override
+		public void open(Configuration conf) {
+			counter = getRuntimeContext().getOperatorState("counter", 0);
+			concat = getRuntimeContext().getOperatorState("concat", "");
+		}
+	}
+	
+	public static class ModKey implements KeySelector<Integer, Serializable> {
+
+		private static final long serialVersionUID = 4193026742083046736L;
+
+		int base;
+
+		public ModKey(int base) {
+			this.base = base;
+		}
+
+		@Override
+		public Integer getKey(Integer value) throws Exception {
+			return value % base;
+		}
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
index 0a26ebb..4e0523e 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
@@ -193,7 +193,7 @@ public class SourceStreamTaskTest extends StreamTaskTestBase {
 		
 		@Override
 		public void open(Configuration conf){
-			state = getRuntimeContext().getOperatorState(1, this);
+			state = getRuntimeContext().getOperatorState("state", 1, this);
 		}
 
 

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
index ea0cb94..344fc7d 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
@@ -157,7 +157,7 @@ public class MockCoContext<IN1, IN2, OUT> {
 		MockCoContext<IN1, IN2, OUT> mockContext = new MockCoContext<IN1, IN2, OUT>(input1, input2);
 		StreamingRuntimeContext runtimeContext = new StreamingRuntimeContext("CoMockTask",
 				new MockEnvironment(3 * 1024 * 1024, new MockInputSplitProvider(), 1024), null,
-				new ExecutionConfig(), null);
+				new ExecutionConfig(), null, null);
 
 		operator.setup(mockContext.collector, runtimeContext);
 

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
index adec338..fc5079a 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
@@ -105,7 +105,7 @@ public class MockContext<IN, OUT> {
 		MockContext<IN, OUT> mockContext = new MockContext<IN, OUT>(inputs);
 		StreamingRuntimeContext runtimeContext = new StreamingRuntimeContext("MockTask",
 				new MockEnvironment(3 * 1024 * 1024, new MockInputSplitProvider(), 1024), null,
-				new ExecutionConfig(), null);
+				new ExecutionConfig(), null, null);
 
 		operator.setup(mockContext.output, runtimeContext);
 		try {

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
index 8c17eec..d58e332 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
@@ -211,7 +211,7 @@ public class StreamCheckpointingITCase {
 			step = getRuntimeContext().getNumberOfParallelSubtasks();
 			
 			
-			index = getRuntimeContext().getOperatorState(getRuntimeContext().getIndexOfThisSubtask());
+			index = getRuntimeContext().getOperatorState("index", getRuntimeContext().getIndexOfThisSubtask());
 			
 			isRunning = true;
 		}
@@ -265,7 +265,7 @@ public class StreamCheckpointingITCase {
 
 		@Override
 		public void open(Configuration conf) {
-			count = getRuntimeContext().getOperatorState(0L);
+			count = getRuntimeContext().getOperatorState("count", 0L);
 		}
 
 		@Override
@@ -347,7 +347,7 @@ public class StreamCheckpointingITCase {
 		
 		@Override
 		public void open(Configuration conf) {
-			this.count = getRuntimeContext().getOperatorState(0L);
+			this.count = getRuntimeContext().getOperatorState("count", 0L);
 		}
 
 		@Override
@@ -369,7 +369,7 @@ public class StreamCheckpointingITCase {
 		
 		@Override
 		public void open(Configuration conf) {
-			this.count = getRuntimeContext().getOperatorState(0L);
+			this.count = getRuntimeContext().getOperatorState("count", 0L);
 		}
 
 		@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
index 6003205..809db1b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
@@ -154,7 +154,7 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
 		
 		@Override
 		public void open(Configuration conf) {
-			collected = getRuntimeContext().getOperatorState(0L);
+			collected = getRuntimeContext().getOperatorState("count", 0L);
 		}
 
 		@Override
@@ -199,7 +199,7 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
 			stepSize = getRuntimeContext().getNumberOfParallelSubtasks();
 			congruence = getRuntimeContext().getIndexOfThisSubtask();
 			toCollect = (end % stepSize > congruence) ? (end / stepSize + 1) : (end / stepSize);
-			collected = getRuntimeContext().getOperatorState(0L);
+			collected = getRuntimeContext().getOperatorState("count", 0L);
 		}
 
 		@Override