You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by gy...@apache.org on 2015/06/25 19:21:39 UTC
[06/12] flink git commit: [streaming] Allow multiple operator states
+ stateful function test rework
[streaming] Allow multiple operator states + stateful function test rework
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ef11e636
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ef11e636
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ef11e636
Branch: refs/heads/master
Commit: ef11e636154c343c6ba557060fb0df3ddcb3c77a
Parents: a7e2458
Author: Gyula Fora <gy...@apache.org>
Authored: Thu May 28 13:31:07 2015 +0200
Committer: Gyula Fora <gy...@apache.org>
Committed: Thu Jun 25 16:38:06 2015 +0200
----------------------------------------------------------------------
.../api/common/functions/RuntimeContext.java | 30 ++-
.../util/AbstractRuntimeUDFContext.java | 4 +-
.../flink/api/common/state/OperatorState.java | 14 +-
.../runtime/state/PartitionedStateStore.java | 4 +
.../api/datastream/IterativeDataStream.java | 2 -
.../api/operators/AbstractStreamOperator.java | 5 +
.../operators/AbstractUdfStreamOperator.java | 34 +--
.../streaming/api/operators/StreamOperator.java | 2 +
.../streaming/api/state/EagerStateStore.java | 11 +-
.../streaming/api/state/LazyStateStore.java | 11 +-
.../state/PartitionedStreamOperatorState.java | 4 +-
.../api/state/StreamOperatorState.java | 9 +-
.../runtime/tasks/OneInputStreamTask.java | 1 +
.../streaming/runtime/tasks/OutputHandler.java | 6 +-
.../streaming/runtime/tasks/StreamTask.java | 29 ++-
.../runtime/tasks/StreamingRuntimeContext.java | 81 +++++--
.../api/state/StatefulFunctionTest.java | 211 -------------------
.../api/state/StatefulOperatorTest.java | 192 +++++++++++++++++
.../runtime/tasks/SourceStreamTaskTest.java | 2 +-
.../flink/streaming/util/MockCoContext.java | 2 +-
.../flink/streaming/util/MockContext.java | 2 +-
.../StreamCheckpointingITCase.java | 8 +-
.../ProcessFailureStreamingRecoveryITCase.java | 4 +-
23 files changed, 369 insertions(+), 299 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
index a3b8f65..2c2ec53 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
@@ -166,10 +166,11 @@ public interface RuntimeContext {
// --------------------------------------------------------------------------------------------
/**
- * Returns the {@link OperatorState} of this operator instance, which can be
- * used to store and update user state in a fault tolerant fashion. The
- * state will be initialized by the provided default value, and the
- * {@link StateCheckpointer} will be used to draw the state snapshots.
+ * Returns the {@link OperatorState} with the given name of the underlying
+ * operator instance, which can be used to store and update user state in a
+ * fault tolerant fashion. The state will be initialized by the provided
+ * default value, and the {@link StateCheckpointer} will be used to draw the
+ * state snapshots.
*
* <p>
* When storing a {@link Serializable} state the user can omit the
@@ -177,6 +178,9 @@ public interface RuntimeContext {
* the snapshot.
* </p>
*
+ * @param name
+ * Identifier for the state allowing that more operator states
+ * can be used by the same operator.
* @param defaultState
* Default value for the operator state. This will be returned
* the first time {@link OperatorState#getState()} (for every
@@ -185,26 +189,30 @@ public interface RuntimeContext {
* @param checkpointer
* The {@link StateCheckpointer} that will be used to draw
* snapshots from the user state.
- * @return The {@link OperatorState} for this instance.
+ * @return The {@link OperatorState} for the underlying operator.
*/
- <S,C extends Serializable> OperatorState<S> getOperatorState(S defaultState, StateCheckpointer<S,C> checkpointer);
+ <S,C extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState, StateCheckpointer<S,C> checkpointer);
/**
- * Returns the {@link OperatorState} of this operator instance, which can be
- * used to store and update user state in a fault tolerant fashion. The
- * state will be initialized by the provided default value.
+ * Returns the {@link OperatorState} with the given name of the underlying
+ * operator instance, which can be used to store and update user state in a
+ * fault tolerant fashion. The state will be initialized by the provided
+ * default value.
*
* <p>
* When storing a non-{@link Serializable} state the user needs to specify a
* {@link StateCheckpointer} for drawing snapshots.
* </p>
*
+ * @param name
+ * Identifier for the state allowing that more operator states can be
+ * used by the same operator.
* @param defaultState
* Default value for the operator state. This will be returned
* the first time {@link OperatorState#getState()} (for every
* state partition) is called before
* {@link OperatorState#updateState(Object)}.
- * @return The {@link OperatorState} for this instance.
+ * @return The {@link OperatorState} for the underlying operator.
*/
- <S extends Serializable> OperatorState<S> getOperatorState(S defaultState);
+ <S extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
index 413565b..9f17bd1 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
@@ -174,12 +174,12 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
}
@Override
- public <S, C extends Serializable> OperatorState<S> getOperatorState(S defaultState, StateCheckpointer<S, C> checkpointer) {
+ public <S, C extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState, StateCheckpointer<S, C> checkpointer) {
throw new UnsupportedOperationException("Operator state is only accessible for streaming operators.");
}
@Override
- public <S extends Serializable> OperatorState<S> getOperatorState(S defaultState) {
+ public <S extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState) {
throw new UnsupportedOperationException("Operator state is only accessible for streaming operators.");
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
index 5b3fa05..9155037 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/state/OperatorState.java
@@ -21,7 +21,7 @@ package org.apache.flink.api.common.state;
import org.apache.flink.api.common.functions.MapFunction;
/**
- * Base class for all streaming operator states. It can represent both
+ * Base interface for all streaming operator states. It can represent both
* partitioned (when state partitioning is defined in the program) or
* non-partitioned user states.
*
@@ -30,7 +30,7 @@ import org.apache.flink.api.common.functions.MapFunction;
* transformation call the operator represents, for instance inside
* {@link MapFunction#map()} and invalid in
* {@link #open(org.apache.flink.configuration.Configuration)} or
- * {@link #close()}.
+ * {@link #close()} methods.
*
* @param <T>
* Type of the operator state
@@ -42,13 +42,7 @@ public interface OperatorState<T> {
* partitioned the returned state is the same for all inputs. If state
* partitioning is applied the state returned depends on the current
* operator input, as the operator maintains an independent state for each
- * partitions.
- *
- * <p>
- * {@link #getState()} returns <code>null</code> if there is no state stored
- * in the operator. This is the expected behaviour before initializing the
- * state with {@link #updateState(T)}.
- * </p>
+ * partition.
*
* @return The operator state corresponding to the current input.
*/
@@ -60,7 +54,7 @@ public interface OperatorState<T> {
* partition) the returned state will represent the updated value.
*
* @param state
- * The updated state.
+ * The new state.
*/
void updateState(T state);
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
index 8b73edf..6353eda 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/PartitionedStateStore.java
@@ -21,6 +21,8 @@ package org.apache.flink.runtime.state;
import java.io.Serializable;
import java.util.Map;
+import org.apache.flink.api.common.state.StateCheckpointer;
+
/**
* Interface for storing and accessing partitioned state. The interface is
* designed in a way that allows implementations for lazily state access.
@@ -43,5 +45,7 @@ public interface PartitionedStateStore<S, C extends Serializable> {
void restoreStates(Map<Serializable, StateHandle<C>> snapshots) throws Exception;
boolean containsKey(Serializable key);
+
+ void setCheckPointer(StateCheckpointer<S, C> checkpointer);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
index 2d70d49..6a48b6a 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/datastream/IterativeDataStream.java
@@ -17,8 +17,6 @@
package org.apache.flink.streaming.api.datastream;
-import org.apache.flink.streaming.api.collector.selector.OutputSelector;
-
/**
* The iterative data stream represents the start of an iteration in a
* {@link DataStream}.
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 40f61d9..cb4bcb0 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -79,4 +79,9 @@ public abstract class AbstractStreamOperator<OUT> implements StreamOperator<OUT>
public void disableInputCopy() {
this.inputCopyDisabled = true;
}
+
+ @Override
+ public StreamingRuntimeContext getRuntimeContext(){
+ return runtimeContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
index cbcbcee..da0fdb7 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/AbstractUdfStreamOperator.java
@@ -19,7 +19,9 @@
package org.apache.flink.streaming.api.operators;
import java.io.Serializable;
+import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.util.FunctionUtils;
@@ -67,24 +69,32 @@ public abstract class AbstractUdfStreamOperator<OUT, F extends Function & Serial
FunctionUtils.closeFunction(userFunction);
}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({ "unchecked", "rawtypes" })
public void restoreInitialState(Serializable state) throws Exception {
- Map<Serializable, StateHandle<Serializable>> snapshots = (Map<Serializable, StateHandle<Serializable>>) state;
+ Map<String, Map<Serializable, StateHandle<Serializable>>> snapshots = (Map<String, Map<Serializable, StateHandle<Serializable>>>) state;
- StreamOperatorState<?, Serializable> operatorState = (StreamOperatorState<?, Serializable>) runtimeContext
- .getOperatorState();
-
- operatorState.restoreState(snapshots);
+ Map<String, StreamOperatorState> operatorStates = runtimeContext.getOperatorStates();
+
+ for (Entry<String, Map<Serializable, StateHandle<Serializable>>> snapshot : snapshots.entrySet()) {
+ StreamOperatorState restoredState = runtimeContext.createRawState();
+ restoredState.restoreState(snapshot.getValue());
+ operatorStates.put(snapshot.getKey(), restoredState);
+ }
}
- public Serializable getStateSnapshotFromFunction(long checkpointId, long timestamp)
- throws Exception {
-
- StreamOperatorState<?,?> operatorState = (StreamOperatorState<?,?>) runtimeContext.getOperatorState();
-
- return (Serializable) operatorState.snapshotState(checkpointId, timestamp);
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public Serializable getStateSnapshotFromFunction(long checkpointId, long timestamp) throws Exception {
+
+ Map<String, StreamOperatorState> operatorStates = runtimeContext.getOperatorStates();
+ Map<String, Map<Serializable, StateHandle<Serializable>>> snapshots = new HashMap<String, Map<Serializable, StateHandle<Serializable>>>();
+
+ for (Entry<String, StreamOperatorState> state : operatorStates.entrySet()) {
+ snapshots.put(state.getKey(), state.getValue().snapshotState(checkpointId, timestamp));
+ }
+
+ return (Serializable) snapshots;
}
public void confirmCheckpointCompleted(long checkpointId, long timestamp,
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index aebff5c..75cea5f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -48,6 +48,8 @@ public interface StreamOperator<OUT> extends Serializable {
* This method is called after no more elements for can arrive for processing.
*/
public void close() throws Exception;
+
+ public StreamingRuntimeContext getRuntimeContext();
/**
* An operator can return true here to disable copying of its input elements. This overrides
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
index 4ac01a5..b374fe4 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/EagerStateStore.java
@@ -30,10 +30,10 @@ import org.apache.flink.runtime.state.StateHandleProvider;
public class EagerStateStore<S, C extends Serializable> implements PartitionedStateStore<S, C> {
- protected StateCheckpointer<S, C> checkpointer;
- protected StateHandleProvider<C> provider;
+ private StateCheckpointer<S, C> checkpointer;
+ private final StateHandleProvider<C> provider;
- Map<Serializable, S> fetchedState;
+ private Map<Serializable, S> fetchedState;
public EagerStateStore(StateCheckpointer<S, C> checkpointer, StateHandleProvider<C> provider) {
this.checkpointer = checkpointer;
@@ -83,4 +83,9 @@ public class EagerStateStore<S, C extends Serializable> implements PartitionedSt
return fetchedState.containsKey(key);
}
+ @Override
+ public void setCheckPointer(StateCheckpointer<S, C> checkpointer) {
+ this.checkpointer = checkpointer;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
index 9872a0c..14484ea 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/LazyStateStore.java
@@ -47,10 +47,10 @@ import org.apache.flink.runtime.state.StateHandleProvider;
public class LazyStateStore<S, C extends Serializable> implements PartitionedStateStore<S, C> {
protected StateCheckpointer<S, C> checkpointer;
- protected StateHandleProvider<C> provider;
+ protected final StateHandleProvider<C> provider;
- Map<Serializable, StateHandle<C>> unfetchedState;
- Map<Serializable, S> fetchedState;
+ private final Map<Serializable, StateHandle<C>> unfetchedState;
+ private final Map<Serializable, S> fetchedState;
public LazyStateStore(StateCheckpointer<S, C> checkpointer, StateHandleProvider<C> provider) {
this.checkpointer = checkpointer;
@@ -114,4 +114,9 @@ public class LazyStateStore<S, C extends Serializable> implements PartitionedSta
return fetchedState.containsKey(key) || unfetchedState.containsKey(key);
}
+ @Override
+ public void setCheckPointer(StateCheckpointer<S, C> checkpointer) {
+ this.checkpointer = checkpointer;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
index 26b2a88..4f0200f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionedStreamOperatorState.java
@@ -45,9 +45,9 @@ public class PartitionedStreamOperatorState<IN, S, C extends Serializable> exten
StreamOperatorState<S, C> {
// KeySelector for getting the state partition key for each input
- private KeySelector<IN, Serializable> keySelector;
+ private final KeySelector<IN, Serializable> keySelector;
- private PartitionedStateStore<S, C> stateStore;
+ private final PartitionedStateStore<S, C> stateStore;
private S defaultState;
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
index 90a3726..bab3421 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/StreamOperatorState.java
@@ -44,7 +44,7 @@ public class StreamOperatorState<S, C extends Serializable> implements OperatorS
private S state;
private StateCheckpointer<S, C> checkpointer;
- private StateHandleProvider<C> provider;
+ private final StateHandleProvider<C> provider;
public StreamOperatorState(StateCheckpointer<S, C> checkpointer, StateHandleProvider<C> provider) {
this.checkpointer = checkpointer;
@@ -66,8 +66,11 @@ public class StreamOperatorState<S, C extends Serializable> implements OperatorS
this.state = state;
}
- public void setDefaultState(S defaultState){
- updateState(defaultState);
+ public void setDefaultState(S defaultState) {
+ // reconsider this as it might cause issues when setting the state to null
+ if (getState() == null) {
+ updateState(defaultState);
+ }
}
public StateCheckpointer<S, C> getCheckpointer() {
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
index 6750b52..87042ba 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
@@ -99,6 +99,7 @@ public class OneInputStreamTask<IN, OUT> extends StreamTask<OUT, OneInputStreamO
StreamRecord<IN> nextRecord;
while (isRunning && (nextRecord = readNext()) != null) {
+ headContext.setNextInput(nextRecord);
streamOperator.processElement(nextRecord.getObject());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
index effce6c..a7755f6 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
@@ -226,9 +226,10 @@ public class OutputHandler<OUT> {
}
private static class OperatorCollector<T> implements Output<T> {
- protected OneInputStreamOperator operator;
- public OperatorCollector(OneInputStreamOperator<?, T> operator) {
+ protected OneInputStreamOperator<Object, T> operator;
+
+ public OperatorCollector(OneInputStreamOperator<Object, T> operator) {
this.operator = operator;
}
@@ -236,6 +237,7 @@ public class OutputHandler<OUT> {
@SuppressWarnings("unchecked")
public void collect(T record) {
try {
+ operator.getRuntimeContext().setNextInput(record);
operator.processElement(record);
} catch (Exception e) {
if (LOG.isErrorEnabled()) {
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index e7f4d9c..4ae72d2 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -42,8 +42,6 @@ import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.StatefulStreamOperator;
import org.apache.flink.streaming.api.operators.StreamOperator;
-import org.apache.flink.streaming.api.state.PartitionedStreamOperatorState;
-import org.apache.flink.streaming.api.state.StreamOperatorState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,6 +64,8 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
protected volatile boolean isRunning = false;
protected List<StreamingRuntimeContext> contexts;
+
+ protected StreamingRuntimeContext headContext;
protected ClassLoader userClassLoader;
@@ -93,7 +93,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
// IterationHead and IterationTail don't have an Operator...
//Create context of the head operator
- StreamingRuntimeContext headContext = createRuntimeContext(configuration);
+ headContext = createRuntimeContext(configuration);
this.contexts.add(headContext);
streamOperator.setup(outputHandler.getOutput(), headContext);
}
@@ -105,19 +105,14 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
return getEnvironment().getTaskName();
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
public StreamingRuntimeContext createRuntimeContext(StreamConfig conf) {
Environment env = getEnvironment();
String operatorName = conf.getStreamOperator(userClassLoader).getClass().getSimpleName();
KeySelector<?,Serializable> statePartitioner = conf.getStatePartitioner(userClassLoader);
- StreamOperatorState state = statePartitioner == null ? new StreamOperatorState(
- getStateHandleProvider()) : new PartitionedStreamOperatorState(
- getStateHandleProvider(), statePartitioner);
-
return new StreamingRuntimeContext(operatorName, env, getUserCodeClassLoader(),
- getExecutionConfig(), state);
+ getExecutionConfig(), statePartitioner, getStateHandleProvider());
}
private StateHandleProvider<Serializable> getStateHandleProvider() {
@@ -216,16 +211,16 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
Serializable headState = chainedStates.get(0);
if (headState != null) {
if (streamOperator instanceof StatefulStreamOperator) {
- ((StatefulStreamOperator) streamOperator).restoreInitialState(headState);
+ ((StatefulStreamOperator<?>) streamOperator).restoreInitialState(headState);
}
}
for (int i = 1; i < chainedStates.size(); i++) {
Serializable chainedState = chainedStates.get(i);
if (chainedState != null) {
- StreamOperator chainedOperator = outputHandler.getChainedOperators().get(i - 1);
+ StreamOperator<?> chainedOperator = outputHandler.getChainedOperators().get(i - 1);
if (chainedOperator instanceof StatefulStreamOperator) {
- ((StatefulStreamOperator) chainedOperator).restoreInitialState(chainedState);
+ ((StatefulStreamOperator<?>) chainedOperator).restoreInitialState(chainedState);
}
}
@@ -233,7 +228,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
} else {
if (streamOperator instanceof StatefulStreamOperator) {
- ((StatefulStreamOperator) streamOperator).restoreInitialState(state);
+ ((StatefulStreamOperator<?>) streamOperator).restoreInitialState(state);
}
}
@@ -254,7 +249,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
Serializable userState = null;
if (streamOperator instanceof StatefulStreamOperator) {
- userState = ((StatefulStreamOperator) streamOperator).getStateSnapshotFromFunction(checkpointId, timestamp);
+ userState = ((StatefulStreamOperator<?>) streamOperator).getStateSnapshotFromFunction(checkpointId, timestamp);
}
@@ -266,7 +261,7 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
for (OneInputStreamOperator<?, ?> chainedOperator : outputHandler.getChainedOperators()) {
if (chainedOperator instanceof StatefulStreamOperator) {
- chainedStates.add(((StatefulStreamOperator) chainedOperator).getStateSnapshotFromFunction(checkpointId, timestamp));
+ chainedStates.add(((StatefulStreamOperator<?>) chainedOperator).getStateSnapshotFromFunction(checkpointId, timestamp));
}
}
@@ -306,13 +301,13 @@ public abstract class StreamTask<OUT, O extends StreamOperator<OUT>> extends Abs
// we do nothing here so far. this should call commit on the source function, for example
synchronized (checkpointLock) {
if (streamOperator instanceof StatefulStreamOperator) {
- ((StatefulStreamOperator) streamOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
+ ((StatefulStreamOperator<?>) streamOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
}
if (hasChainedOperators) {
for (OneInputStreamOperator<?, ?> chainedOperator : outputHandler.getChainedOperators()) {
if (chainedOperator instanceof StatefulStreamOperator) {
- ((StatefulStreamOperator) chainedOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
+ ((StatefulStreamOperator<?>) chainedOperator).confirmCheckpointCompleted(checkpointId, timestamp, null);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
index 5fd158c..77639a0 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
@@ -19,35 +19,44 @@
package org.apache.flink.streaming.runtime.tasks;
import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
import org.apache.flink.api.common.state.OperatorState;
import org.apache.flink.api.common.state.StateCheckpointer;
+import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.operators.util.TaskConfig;
+import org.apache.flink.runtime.state.StateHandleProvider;
+import org.apache.flink.streaming.api.state.PartitionedStreamOperatorState;
import org.apache.flink.streaming.api.state.StreamOperatorState;
/**
* Implementation of the {@link RuntimeContext}, created by runtime stream UDF
* operators.
*/
+@SuppressWarnings("rawtypes")
public class StreamingRuntimeContext extends RuntimeUDFContext {
private final Environment env;
- @SuppressWarnings("rawtypes")
- private StreamOperatorState state;
-
+ private final Map<String, StreamOperatorState> states;
+ private final KeySelector<?, ?> statePartitioner;
+ private final StateHandleProvider<?> provider;
public StreamingRuntimeContext(String name, Environment env, ClassLoader userCodeClassLoader,
- ExecutionConfig executionConfig, StreamOperatorState<?, ?> state) {
+ ExecutionConfig executionConfig, KeySelector<?, ?> statePartitioner,
+ StateHandleProvider<?> provider) {
super(name, env.getNumberOfSubtasks(), env.getIndexInSubtaskGroup(), userCodeClassLoader,
executionConfig, env.getDistributedCacheEntries());
this.env = env;
- this.state = state;
+ this.statePartitioner = statePartitioner;
+ this.states = new HashMap<String, StreamOperatorState>();
+ this.provider = provider;
}
/**
@@ -68,25 +77,73 @@ public class StreamingRuntimeContext extends RuntimeUDFContext {
public Configuration getTaskStubParameters() {
return new TaskConfig(env.getTaskConfiguration()).getStubParameters();
}
-
+
@SuppressWarnings("unchecked")
@Override
- public <S, C extends Serializable> OperatorState<S> getOperatorState(S defaultState,
- StateCheckpointer<S, C> checkpointer) {
+ public <S, C extends Serializable> OperatorState<S> getOperatorState(String name,
+ S defaultState, StateCheckpointer<S, C> checkpointer) {
+ StreamOperatorState state;
+ if (states.containsKey(name)) {
+ state = states.get(name);
+ } else {
+ state = createRawState();
+ states.put(name, state);
+ }
+ state.setDefaultState(defaultState);
state.setCheckpointer(checkpointer);
+
return (OperatorState<S>) state;
}
-
+
@SuppressWarnings("unchecked")
@Override
- public <S extends Serializable> OperatorState<S> getOperatorState(S defaultState) {
+ public <S extends Serializable> OperatorState<S> getOperatorState(String name, S defaultState) {
+ StreamOperatorState state;
+ if (states.containsKey(name)) {
+ state = states.get(name);
+ } else {
+ state = createRawState();
+ states.put(name, state);
+ }
state.setDefaultState(defaultState);
+
return (OperatorState<S>) state;
}
+
+ /**
+ * Creates an empty state depending on the partitioning state.
+ *
+ * @return An empty operator state.
+ */
+ @SuppressWarnings("unchecked")
+ public StreamOperatorState createRawState() {
+ if (statePartitioner == null) {
+ return new StreamOperatorState(provider);
+ } else {
+ return new PartitionedStreamOperatorState(provider, statePartitioner);
+ }
+ }
+ /**
+ * Provides access to the all the states contained in the context
+ *
+ * @return All the states for the underlying operator.
+ */
+ public Map<String, StreamOperatorState> getOperatorStates() {
+ return states;
+ }
+
+ /**
+ * Sets the next input of the underlying operators, used to access partitioned states.
+ * @param nextRecord Next input of the operator.
+ */
@SuppressWarnings("unchecked")
- public <S extends Serializable> OperatorState<S> getOperatorState() {
- return (OperatorState<S>) state;
+ public void setNextInput(Object nextRecord) {
+ if (statePartitioner != null) {
+ for (StreamOperatorState state : states.values()) {
+ ((PartitionedStreamOperatorState) state).setCurrentInput(nextRecord);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java
deleted file mode 100644
index f4c1a89..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulFunctionTest.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.state;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.DoubleCounter;
-import org.apache.flink.api.common.accumulators.Histogram;
-import org.apache.flink.api.common.accumulators.IntCounter;
-import org.apache.flink.api.common.accumulators.LongCounter;
-import org.apache.flink.api.common.cache.DistributedCache;
-import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.api.common.state.OperatorState;
-import org.apache.flink.api.common.state.StateCheckpointer;
-import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.state.LocalStateHandle.LocalStateHandleProvider;
-import org.apache.flink.runtime.state.StateHandle;
-import org.apache.flink.shaded.com.google.common.collect.ImmutableMap;
-import org.apache.flink.util.InstantiationUtil;
-import org.junit.Test;
-
-public class StatefulFunctionTest {
-
- @SuppressWarnings("unchecked")
- @Test
- public void simpleStateTest() throws Exception {
-
- StatefulMapper mapper = new StatefulMapper();
- MockContext context = new MockContext(false, mapper);
- mapper.setRuntimeContext(context);
- mapper.open(null);
-
- assertEquals(Arrays.asList("1", "2", "3", "4", "5"),
- applyOnSequence(mapper, 1, 5, context.state));
- assertEquals((Integer) 5, context.state.getState());
-
- byte[] serializedState = InstantiationUtil.serializeObject(context.state
- .snapshotState(1, 1));
-
- StatefulMapper restoredMapper = new StatefulMapper();
- MockContext restoredContext = new MockContext(false, restoredMapper);
- restoredMapper.setRuntimeContext(context);
- restoredMapper.open(null);
-
- assertEquals(null, restoredContext.state.getState());
-
- Map<Serializable, StateHandle<Integer>> deserializedState = (Map<Serializable, StateHandle<Integer>>) InstantiationUtil
- .deserializeObject(serializedState, Thread.currentThread().getContextClassLoader());
-
- restoredContext.state.restoreState(deserializedState);
-
- assertEquals((Integer) 5, restoredContext.state.getState());
-
- }
-
- @SuppressWarnings("unchecked")
- @Test
- public void partitionedStateTest() throws Exception {
- StatefulMapper mapper = new StatefulMapper();
- MockContext context = new MockContext(true, mapper);
- mapper.setRuntimeContext(context);
- mapper.open(null);
-
- assertEquals(Arrays.asList("1", "2", "3", "4", "5"),
- applyOnSequence(mapper, 1, 5, context.state));
- assertEquals(ImmutableMap.of(0, 2, 1, 3), context.state.getPartitionedState());
-
- byte[] serializedState = InstantiationUtil.serializeObject(context.state
- .snapshotState(1, 1));
-
- StatefulMapper restoredMapper = new StatefulMapper();
- MockContext restoredContext = new MockContext(true, restoredMapper);
- restoredMapper.setRuntimeContext(context);
- restoredMapper.open(null);
-
- assertEquals(null, restoredContext.state.getState());
-
- Map<Serializable, StateHandle<Integer>> deserializedState = (Map<Serializable, StateHandle<Integer>>) InstantiationUtil
- .deserializeObject(serializedState, Thread.currentThread().getContextClassLoader());
-
- restoredContext.state.restoreState(deserializedState);
-
- assertEquals(ImmutableMap.of(0, 2, 1, 3), restoredContext.state.getPartitionedState());
-
- }
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- private <T> List<T> applyOnSequence(MapFunction<Integer, T> mapper, int from, int to,
- StreamOperatorState state) throws Exception {
- List<T> output = new ArrayList<T>();
- for (int i = from; i <= to; i++) {
- if (state instanceof PartitionedStreamOperatorState) {
- ((PartitionedStreamOperatorState) state).setCurrentInput(i);
- }
- output.add(mapper.map(i));
- }
- return output;
- }
-
- public static class ModKey implements KeySelector<Integer, Serializable> {
-
- private static final long serialVersionUID = 4193026742083046736L;
-
- int base;
-
- public ModKey(int base) {
- this.base = base;
- }
-
- @Override
- public Integer getKey(Integer value) throws Exception {
- return value % base;
- }
-
- }
-
- public static class StatefulMapper extends RichMapFunction<Integer, String> {
-
- private static final long serialVersionUID = -9007873655253339356L;
- OperatorState<Integer> opState;
-
- @Override
- public String map(Integer value) throws Exception {
- opState.updateState(opState.getState() + 1);
- return value.toString();
- }
-
- @Override
- public void open(Configuration conf) {
- opState = getRuntimeContext().getOperatorState(0);
- }
- }
-
- public static class MockContext implements RuntimeContext {
-
- StreamOperatorState<Integer, Integer> state;
-
- public MockContext(boolean isPartitionedState, StatefulMapper mapper) {
- if (isPartitionedState) {
- this.state = new PartitionedStreamOperatorState<Integer, Integer, Integer>(
- new LocalStateHandleProvider<Integer>(), new ModKey(2));
- } else {
- this.state = new StreamOperatorState<Integer, Integer>(
- new LocalStateHandleProvider<Integer>());
- }
- }
-
- public String getTaskName() {return null;}
- public int getNumberOfParallelSubtasks() {return 0;}
- public int getIndexOfThisSubtask() {return 0;}
- public ExecutionConfig getExecutionConfig() {return null;}
- public ClassLoader getUserCodeClassLoader() {return null;}
- public <V, A extends Serializable> void addAccumulator(String name, Accumulator<V, A> accumulator) {}
- public <V, A extends Serializable> Accumulator<V, A> getAccumulator(String name) {return null;}
- public HashMap<String, Accumulator<?, ?>> getAllAccumulators() {return null;}
- public IntCounter getIntCounter(String name) {return null;}
- public LongCounter getLongCounter(String name) {return null;}
- public DoubleCounter getDoubleCounter(String name) {return null;}
- public Histogram getHistogram(String name) {return null;}
- public <RT> List<RT> getBroadcastVariable(String name) {return null;}
- public <T, C> C getBroadcastVariableWithInitializer(String name,
- BroadcastVariableInitializer<T, C> initializer) {return null;}
- public DistributedCache getDistributedCache() {return null;}
-
- @SuppressWarnings("unchecked")
- @Override
- public <S, C extends Serializable> OperatorState<S> getOperatorState(S defaultState,
- StateCheckpointer<S, C> checkpointer) {
- state.setDefaultState((Integer) defaultState);
- return (OperatorState<S>) state;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public <S extends Serializable> OperatorState<S> getOperatorState(S defaultState) {
- state.setDefaultState((Integer) defaultState);
- return (OperatorState<S>) state;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java
new file mode 100644
index 0000000..99015c9
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/StatefulOperatorTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.state;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.state.OperatorState;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
+import org.apache.flink.runtime.operators.testutils.MockInputSplitProvider;
+import org.apache.flink.runtime.state.LocalStateHandle.LocalStateHandleProvider;
+import org.apache.flink.shaded.com.google.common.collect.ImmutableMap;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.operators.StreamMap;
+import org.apache.flink.streaming.runtime.tasks.StreamingRuntimeContext;
+import org.apache.flink.util.InstantiationUtil;
+import org.junit.Test;
+
+/**
+ * Test the functionality supported by stateful user functions for both
+ * partitioned and non-partitioned user states. This test mimics the runtime
+ * behavior of stateful stream operators.
+ */
+public class StatefulOperatorTest {
+
+ @Test
+ public void simpleStateTest() throws Exception {
+
+ List<String> out = new ArrayList<String>();
+
+ StreamMap<Integer, String> map = createOperatorWithContext(out, null, null);
+ StreamingRuntimeContext context = map.getRuntimeContext();
+
+ processInputs(map, Arrays.asList(1, 2, 3, 4, 5));
+
+ assertEquals(Arrays.asList("1", "2", "3", "4", "5"), out);
+ assertEquals((Integer) 5, context.getOperatorState("counter", 0).getState());
+ assertEquals("12345", context.getOperatorState("concat", "").getState());
+
+ byte[] serializedState = InstantiationUtil.serializeObject(map.getStateSnapshotFromFunction(1, 1));
+
+ StreamMap<Integer, String> restoredMap = createOperatorWithContext(out, null, serializedState);
+ StreamingRuntimeContext restoredContext = restoredMap.getRuntimeContext();
+
+ assertEquals((Integer) 5, restoredContext.getOperatorState("counter", 0).getState());
+ assertEquals("12345", restoredContext.getOperatorState("concat", "").getState());
+ out.clear();
+
+ processInputs(restoredMap, Arrays.asList(7, 8));
+
+ assertEquals(Arrays.asList("7", "8"), out);
+ assertEquals((Integer) 7, restoredContext.getOperatorState("counter", 0).getState());
+ assertEquals("1234578", restoredContext.getOperatorState("concat", "").getState());
+
+ }
+
+ @Test
+ public void partitionedStateTest() throws Exception {
+ List<String> out = new ArrayList<String>();
+
+ StreamMap<Integer, String> map = createOperatorWithContext(out, new ModKey(2), null);
+ StreamingRuntimeContext context = map.getRuntimeContext();
+
+ processInputs(map, Arrays.asList(1, 2, 3, 4, 5));
+
+ assertEquals(Arrays.asList("1", "2", "3", "4", "5"), out);
+ assertEquals(ImmutableMap.of(0, 2, 1, 3), context.getOperatorStates().get("counter").getPartitionedState());
+ assertEquals(ImmutableMap.of(0, "24", 1, "135"), context.getOperatorStates().get("concat")
+ .getPartitionedState());
+
+ byte[] serializedState = InstantiationUtil.serializeObject(map.getStateSnapshotFromFunction(1, 1));
+
+ StreamMap<Integer, String> restoredMap = createOperatorWithContext(out, new ModKey(2), serializedState);
+ StreamingRuntimeContext restoredContext = restoredMap.getRuntimeContext();
+
+ assertEquals(ImmutableMap.of(0, 2, 1, 3), restoredContext.getOperatorStates().get("counter")
+ .getPartitionedState());
+ assertEquals(ImmutableMap.of(0, "24", 1, "135"), restoredContext.getOperatorStates().get("concat")
+ .getPartitionedState());
+ out.clear();
+
+ processInputs(restoredMap, Arrays.asList(7, 8));
+
+ assertEquals(Arrays.asList("7", "8"), out);
+ assertEquals(ImmutableMap.of(0, 3, 1, 4), restoredContext.getOperatorStates().get("counter")
+ .getPartitionedState());
+ assertEquals(ImmutableMap.of(0, "248", 1, "1357"), restoredContext.getOperatorStates().get("concat")
+ .getPartitionedState());
+
+ }
+
+ private void processInputs(StreamMap<Integer, ?> map, List<Integer> input) throws Exception {
+ for (Integer i : input) {
+ map.getRuntimeContext().setNextInput(i);
+ map.processElement(i);
+ }
+ }
+
+ private StreamMap<Integer, String> createOperatorWithContext(List<String> output,
+ KeySelector<Integer, Serializable> partitioner, byte[] serializedState) throws Exception {
+ final List<String> outputList = output;
+
+ StreamingRuntimeContext context = new StreamingRuntimeContext("MockTask", new MockEnvironment(3 * 1024 * 1024,
+ new MockInputSplitProvider(), 1024), null, new ExecutionConfig(), partitioner,
+ new LocalStateHandleProvider<Serializable>());
+
+ StreamMap<Integer, String> op = new StreamMap<Integer, String>(new StatefulMapper());
+
+ op.setup(new Output<String>() {
+
+ @Override
+ public void collect(String record) {
+ outputList.add(record);
+ }
+
+ @Override
+ public void close() {
+ }
+ }, context);
+
+ if (serializedState != null) {
+ op.restoreInitialState((Serializable) InstantiationUtil.deserializeObject(serializedState, Thread
+ .currentThread().getContextClassLoader()));
+ }
+
+ op.open(null);
+
+ return op;
+ }
+
+ public static class StatefulMapper extends RichMapFunction<Integer, String> {
+
+ private static final long serialVersionUID = -9007873655253339356L;
+ OperatorState<Integer> counter;
+ OperatorState<String> concat;
+
+ @Override
+ public String map(Integer value) throws Exception {
+ counter.updateState(counter.getState() + 1);
+ concat.updateState(concat.getState() + value.toString());
+ return value.toString();
+ }
+
+ @Override
+ public void open(Configuration conf) {
+ counter = getRuntimeContext().getOperatorState("counter", 0);
+ concat = getRuntimeContext().getOperatorState("concat", "");
+ }
+ }
+
+ public static class ModKey implements KeySelector<Integer, Serializable> {
+
+ private static final long serialVersionUID = 4193026742083046736L;
+
+ int base;
+
+ public ModKey(int base) {
+ this.base = base;
+ }
+
+ @Override
+ public Integer getKey(Integer value) throws Exception {
+ return value % base;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
index 0a26ebb..4e0523e 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceStreamTaskTest.java
@@ -193,7 +193,7 @@ public class SourceStreamTaskTest extends StreamTaskTestBase {
@Override
public void open(Configuration conf){
- state = getRuntimeContext().getOperatorState(1, this);
+ state = getRuntimeContext().getOperatorState("state", 1, this);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
index ea0cb94..344fc7d 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockCoContext.java
@@ -157,7 +157,7 @@ public class MockCoContext<IN1, IN2, OUT> {
MockCoContext<IN1, IN2, OUT> mockContext = new MockCoContext<IN1, IN2, OUT>(input1, input2);
StreamingRuntimeContext runtimeContext = new StreamingRuntimeContext("CoMockTask",
new MockEnvironment(3 * 1024 * 1024, new MockInputSplitProvider(), 1024), null,
- new ExecutionConfig(), null);
+ new ExecutionConfig(), null, null);
operator.setup(mockContext.collector, runtimeContext);
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
index adec338..fc5079a 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/util/MockContext.java
@@ -105,7 +105,7 @@ public class MockContext<IN, OUT> {
MockContext<IN, OUT> mockContext = new MockContext<IN, OUT>(inputs);
StreamingRuntimeContext runtimeContext = new StreamingRuntimeContext("MockTask",
new MockEnvironment(3 * 1024 * 1024, new MockInputSplitProvider(), 1024), null,
- new ExecutionConfig(), null);
+ new ExecutionConfig(), null, null);
operator.setup(mockContext.output, runtimeContext);
try {
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
index 8c17eec..d58e332 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
@@ -211,7 +211,7 @@ public class StreamCheckpointingITCase {
step = getRuntimeContext().getNumberOfParallelSubtasks();
- index = getRuntimeContext().getOperatorState(getRuntimeContext().getIndexOfThisSubtask());
+ index = getRuntimeContext().getOperatorState("index", getRuntimeContext().getIndexOfThisSubtask());
isRunning = true;
}
@@ -265,7 +265,7 @@ public class StreamCheckpointingITCase {
@Override
public void open(Configuration conf) {
- count = getRuntimeContext().getOperatorState(0L);
+ count = getRuntimeContext().getOperatorState("count", 0L);
}
@Override
@@ -347,7 +347,7 @@ public class StreamCheckpointingITCase {
@Override
public void open(Configuration conf) {
- this.count = getRuntimeContext().getOperatorState(0L);
+ this.count = getRuntimeContext().getOperatorState("count", 0L);
}
@Override
@@ -369,7 +369,7 @@ public class StreamCheckpointingITCase {
@Override
public void open(Configuration conf) {
- this.count = getRuntimeContext().getOperatorState(0L);
+ this.count = getRuntimeContext().getOperatorState("count", 0L);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/ef11e636/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
index 6003205..809db1b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
@@ -154,7 +154,7 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
@Override
public void open(Configuration conf) {
- collected = getRuntimeContext().getOperatorState(0L);
+ collected = getRuntimeContext().getOperatorState("count", 0L);
}
@Override
@@ -199,7 +199,7 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
stepSize = getRuntimeContext().getNumberOfParallelSubtasks();
congruence = getRuntimeContext().getIndexOfThisSubtask();
toCollect = (end % stepSize > congruence) ? (end / stepSize + 1) : (end / stepSize);
- collected = getRuntimeContext().getOperatorState(0L);
+ collected = getRuntimeContext().getOperatorState("count", 0L);
}
@Override