You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2016/01/12 21:01:28 UTC
[08/10] storm git commit: Package name fixes
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/DefaultStateSerializer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/DefaultStateSerializer.java b/storm-core/src/jvm/org/apache/storm/state/DefaultStateSerializer.java
new file mode 100644
index 0000000..55e934e
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/DefaultStateSerializer.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A default implementation that uses Kryo to serialize and de-serialize
+ * the state.
+ */
+public class DefaultStateSerializer<T> implements Serializer<T> {
+ private final Kryo kryo;
+ private final Output output;
+
+ /**
+ * Constructs a {@link DefaultStateSerializer} instance with the given list
+ * of classes registered in kryo.
+ *
+ * @param classesToRegister the classes to register.
+ */
+ public DefaultStateSerializer(List<Class<?>> classesToRegister) {
+ kryo = new Kryo();
+ output = new Output(2000, 2000000000);
+ for (Class<?> klazz : classesToRegister) {
+ kryo.register(klazz);
+ }
+ }
+
+ public DefaultStateSerializer() {
+ this(Collections.<Class<?>>emptyList());
+ }
+
+ @Override
+ public byte[] serialize(T obj) {
+ output.clear();
+ kryo.writeClassAndObject(output, obj);
+ return output.toBytes();
+ }
+
+ @Override
+ public T deserialize(byte[] b) {
+ Input input = new Input(b);
+ return (T) kryo.readClassAndObject(input);
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java b/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java
new file mode 100644
index 0000000..4b116ba
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueState.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * An in-memory implementation of the {@link State}
+ */
+public class InMemoryKeyValueState<K, V> implements KeyValueState<K, V> {
+ private static final Logger LOG = LoggerFactory.getLogger(InMemoryKeyValueState.class);
+ private static final long DEFAULT_TXID = -1;
+ private TxIdState<K, V> commitedState;
+ private TxIdState<K, V> preparedState;
+ private Map<K, V> state = new ConcurrentHashMap<>();
+
+ private static class TxIdState<K, V> {
+ private long txid;
+ private Map<K, V> state;
+
+ TxIdState(long txid, Map<K, V> state) {
+ this.txid = txid;
+ this.state = state;
+ }
+
+ @Override
+ public String toString() {
+ return "TxIdState{" +
+ "txid=" + txid +
+ ", state=" + state +
+ '}';
+ }
+ }
+
+ @Override
+ public void put(K key, V value) {
+ state.put(key, value);
+ }
+
+ @Override
+ public V get(K key) {
+ return state.get(key);
+ }
+
+ @Override
+ public V get(K key, V defaultValue) {
+ V val = get(key);
+ return val != null ? val : defaultValue;
+ }
+
+ @Override
+ public void commit() {
+ commitedState = new TxIdState<>(DEFAULT_TXID, new ConcurrentHashMap<>(state));
+ }
+
+ @Override
+ public void prepareCommit(long txid) {
+ LOG.debug("prepare commit, txid {}", txid);
+ if (preparedState != null && txid > preparedState.txid) {
+ throw new RuntimeException("Cannot prepare a new txn while there is a pending txn");
+ }
+ preparedState = new TxIdState<>(txid, new ConcurrentHashMap<K, V>(state));
+ }
+
+ @Override
+ public void commit(long txid) {
+ LOG.debug("commit, txid {}", txid);
+ if (preparedState != null && txid == preparedState.txid) {
+ commitedState = preparedState;
+ preparedState = null;
+ } else {
+ throw new RuntimeException("Invalid prepared state for commit, " +
+ "preparedState " + preparedState + " txid " + txid);
+ }
+ }
+
+ @Override
+ public void rollback() {
+ preparedState = null;
+ if (commitedState != null) {
+ state = commitedState.state;
+ } else {
+ state = new ConcurrentHashMap<>();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "InMemoryKeyValueState{" +
+ "commitedState=" + commitedState +
+ ", preparedState=" + preparedState +
+ ", state=" + state +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java b/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java
new file mode 100644
index 0000000..bc33241
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/InMemoryKeyValueStateProvider.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import org.apache.storm.task.TopologyContext;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Provides {@link InMemoryKeyValueState}
+ */
+public class InMemoryKeyValueStateProvider implements StateProvider {
+ private final ConcurrentHashMap<String, State> states = new ConcurrentHashMap<>();
+
+ @Override
+ public State newState(String namespace, Map stormConf, TopologyContext context) {
+ State state = states.get(namespace);
+ if (state == null) {
+ State newState = new InMemoryKeyValueState<>();
+ state = states.putIfAbsent(namespace, newState);
+ if (state == null) {
+ state = newState;
+ }
+ }
+ return state;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/KeyValueState.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/KeyValueState.java b/storm-core/src/jvm/org/apache/storm/state/KeyValueState.java
new file mode 100644
index 0000000..3ab60f1
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/KeyValueState.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+/**
+ * A state that supports key-value mappings.
+ */
+public interface KeyValueState<K, V> extends State {
+ /**
+ * Maps the value with the key
+ *
+ * @param key the key
+ * @param value the value
+ */
+ void put(K key, V value);
+
+ /**
+ * Returns the value mapped to the key
+ *
+ * @param key the key
+ * @return the value or null if no mapping is found
+ */
+ V get(K key);
+
+ /**
+ * Returns the value mapped to the key or defaultValue if no mapping is found.
+ *
+ * @param key the key
+ * @param defaultValue the value to return if no mapping is found
+ * @return the value or defaultValue if no mapping is found
+ */
+ V get(K key, V defaultValue);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/Serializer.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/Serializer.java b/storm-core/src/jvm/org/apache/storm/state/Serializer.java
new file mode 100644
index 0000000..b442cec
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/Serializer.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import java.io.Serializable;
+
+/**
+ * Interface to be implemented for serlializing and de-serializing the
+ * state.
+ */
+public interface Serializer<T> extends Serializable {
+ byte[] serialize(T obj);
+
+ T deserialize(byte[] b);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/State.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/State.java b/storm-core/src/jvm/org/apache/storm/state/State.java
new file mode 100644
index 0000000..2a7505c
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/State.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import org.apache.storm.topology.IStatefulBolt;
+
+/**
+ * The state of the component that is either managed by the framework (e.g in case of {@link IStatefulBolt})
+ * or managed by the the individual components themselves.
+ */
+public interface State {
+ /**
+ * Invoked by the framework to prepare a transaction for commit. It should be possible
+ * to commit the prepared state later.
+ * <p>
+ * The same txid can be prepared again, but the next txid cannot be prepared
+ * when previous one is not yet committed.
+ * </p>
+ *
+ * @param txid the transaction id
+ */
+ void prepareCommit(long txid);
+
+ /**
+ * Commit a previously prepared transaction. It should be possible to retrieve a committed state later.
+ *
+ * @param txid the transaction id
+ */
+ void commit(long txid);
+
+ /**
+ * Persist the current state. This is used when the component manages the state.
+ */
+ void commit();
+
+ /**
+ * Rollback a prepared transaction to the previously committed state.
+ */
+ void rollback();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/StateFactory.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/StateFactory.java b/storm-core/src/jvm/org/apache/storm/state/StateFactory.java
new file mode 100644
index 0000000..d5ec7ae
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/StateFactory.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import org.apache.storm.Config;
+import org.apache.storm.task.TopologyContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * A factory for creating {@link State} instances
+ */
+public class StateFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(StateFactory.class);
+
+ private static final String DEFAULT_PROVIDER = "org.apache.storm.state.InMemoryKeyValueStateProvider";
+
+ /**
+ * Returns a new state instance using the {@link Config#TOPOLOGY_STATE_PROVIDER} or a
+ * {@link InMemoryKeyValueState} if no provider is configured.
+ *
+ * @param namespace the state namespace
+ * @param stormConf the storm conf
+ * @param context the topology context
+ * @return the state instance
+ */
+ public static State getState(String namespace, Map stormConf, TopologyContext context) {
+ State state;
+ try {
+ String provider = null;
+ if (stormConf.containsKey(Config.TOPOLOGY_STATE_PROVIDER)) {
+ provider = (String) stormConf.get(Config.TOPOLOGY_STATE_PROVIDER);
+ } else {
+ provider = DEFAULT_PROVIDER;
+ }
+ Class<?> klazz = Class.forName(provider);
+ Object object = klazz.newInstance();
+ if (object instanceof StateProvider) {
+ state = ((StateProvider) object).newState(namespace, stormConf, context);
+ } else {
+ String msg = "Invalid state provider '" + provider +
+ "'. Should implement org.apache.storm.state.StateProvider";
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+ } catch (Exception ex) {
+ LOG.error("Got exception while loading the state provider", ex);
+ throw new RuntimeException(ex);
+ }
+ return state;
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/state/StateProvider.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/state/StateProvider.java b/storm-core/src/jvm/org/apache/storm/state/StateProvider.java
new file mode 100644
index 0000000..0dad13e
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/state/StateProvider.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import org.apache.storm.task.TopologyContext;
+
+import java.util.Map;
+
+/**
+ * Used by the {@link StateFactory} to create a new state instances.
+ */
+public interface StateProvider {
+ /**
+ * Returns a new state instance. Each state belongs unique namespace which is typically
+ * the componentid-task of the task, so that each task can have its own unique state.
+ *
+ * @param namespace a namespace of the state
+ * @param stormConf the storm topology configuration
+ * @param context the {@link TopologyContext}
+ * @return a previously saved state instance
+ */
+ State newState(String namespace, Map stormConf, TopologyContext context);
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java b/storm-core/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java
new file mode 100644
index 0000000..675be57
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/topology/CheckpointTupleForwarder.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.topology;
+
+import org.apache.storm.generated.GlobalStreamId;
+import org.apache.storm.spout.CheckpointSpout;
+import org.apache.storm.task.IOutputCollector;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.tuple.Fields;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.storm.spout.CheckPointState.Action;
+import static org.apache.storm.spout.CheckPointState.Action.ROLLBACK;
+import static org.apache.storm.spout.CheckpointSpout.*;
+
+/**
+ * Wraps {@link IRichBolt} and forwards checkpoint tuples in a
+ * stateful topology.
+ * <p>
+ * When a storm topology contains one or more {@link IStatefulBolt} all non-stateful
+ * bolts are wrapped in {@link CheckpointTupleForwarder} so that the checkpoint tuples
+ * can flow through the entire topology DAG.
+ * </p>
+ */
+public class CheckpointTupleForwarder implements IRichBolt {
+ private static final Logger LOG = LoggerFactory.getLogger(CheckpointTupleForwarder.class);
+ private final IRichBolt bolt;
+ private final Map<TransactionRequest, Integer> transactionRequestCount;
+ private int checkPointInputTaskCount;
+ private long lastTxid = Long.MIN_VALUE;
+ protected AnchoringOutputCollector collector;
+
+ public CheckpointTupleForwarder(IRichBolt bolt) {
+ this.bolt = bolt;
+ transactionRequestCount = new HashMap<>();
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ this.collector = new AnchoringOutputCollector(collector);
+ bolt.prepare(stormConf, context, this.collector);
+ checkPointInputTaskCount = getCheckpointInputTaskCount(context);
+ }
+
+ @Override
+ public void execute(Tuple input) {
+ if (CheckpointSpout.isCheckpoint(input)) {
+ processCheckpoint(input);
+ } else {
+ handleTuple(input);
+ }
+ }
+
+ @Override
+ public void cleanup() {
+ bolt.cleanup();
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ bolt.declareOutputFields(declarer);
+ declarer.declareStream(CHECKPOINT_STREAM_ID, new Fields(CHECKPOINT_FIELD_TXID, CHECKPOINT_FIELD_ACTION));
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return bolt.getComponentConfiguration();
+ }
+
+ /**
+ * Forwards the checkpoint tuple downstream. Sub-classes can override
+ * with the logic for handling checkpoint tuple.
+ *
+ * @param checkpointTuple the checkpoint tuple
+ * @param action the action (prepare, commit, rollback or initstate)
+ * @param txid the transaction id.
+ */
+ protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) {
+ collector.emit(CHECKPOINT_STREAM_ID, checkpointTuple, new Values(txid, action));
+ collector.ack(checkpointTuple);
+ }
+
+ /**
+ * Hands off tuple to the wrapped bolt to execute. Sub-classes can
+ * override the behavior.
+ * <p>
+ * Right now tuples continue to get forwarded while waiting for checkpoints to arrive on other streams
+ * after checkpoint arrives on one of the streams. This can cause duplicates but still at least once.
+ * </p>
+ *
+ * @param input the input tuple
+ */
+ protected void handleTuple(Tuple input) {
+ collector.setContext(input);
+ bolt.execute(input);
+ collector.ack(input);
+ }
+
+ /**
+ * Invokes handleCheckpoint once checkpoint tuple is received on
+ * all input checkpoint streams to this component.
+ */
+ private void processCheckpoint(Tuple input) {
+ Action action = (Action) input.getValueByField(CHECKPOINT_FIELD_ACTION);
+ long txid = input.getLongByField(CHECKPOINT_FIELD_TXID);
+ if (shouldProcessTransaction(action, txid)) {
+ LOG.debug("Processing action {}, txid {}", action, txid);
+ try {
+ if (txid >= lastTxid) {
+ handleCheckpoint(input, action, txid);
+ if (action == ROLLBACK) {
+ lastTxid = txid - 1;
+ } else {
+ lastTxid = txid;
+ }
+ } else {
+ LOG.debug("Ignoring old transaction. Action {}, txid {}", action, txid);
+ collector.ack(input);
+ }
+ } catch (Throwable th) {
+ LOG.error("Got error while processing checkpoint tuple", th);
+ collector.fail(input);
+ collector.reportError(th);
+ }
+ } else {
+ LOG.debug("Waiting for action {}, txid {} from all input tasks. checkPointInputTaskCount {}, " +
+ "transactionRequestCount {}", action, txid, checkPointInputTaskCount, transactionRequestCount);
+ collector.ack(input);
+ }
+ }
+
+ /**
+ * returns the total number of input checkpoint streams across
+ * all input tasks to this component.
+ */
+ private int getCheckpointInputTaskCount(TopologyContext context) {
+ int count = 0;
+ for (GlobalStreamId inputStream : context.getThisSources().keySet()) {
+ if (CHECKPOINT_STREAM_ID.equals(inputStream.get_streamId())) {
+ count += context.getComponentTasks(inputStream.get_componentId()).size();
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Checks if check points have been received from all tasks across
+ * all input streams to this component
+ */
+ private boolean shouldProcessTransaction(Action action, long txid) {
+ TransactionRequest request = new TransactionRequest(action, txid);
+ Integer count;
+ if ((count = transactionRequestCount.get(request)) == null) {
+ transactionRequestCount.put(request, 1);
+ count = 1;
+ } else {
+ transactionRequestCount.put(request, ++count);
+ }
+ if (count == checkPointInputTaskCount) {
+ transactionRequestCount.remove(request);
+ return true;
+ }
+ return false;
+ }
+
+ private static class TransactionRequest {
+ private final Action action;
+ private final long txid;
+
+ TransactionRequest(Action action, long txid) {
+ this.action = action;
+ this.txid = txid;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TransactionRequest that = (TransactionRequest) o;
+
+ if (txid != that.txid) return false;
+ return !(action != null ? !action.equals(that.action) : that.action != null);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = action != null ? action.hashCode() : 0;
+ result = 31 * result + (int) (txid ^ (txid >>> 32));
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "TransactionRequest{" +
+ "action='" + action + '\'' +
+ ", txid=" + txid +
+ '}';
+ }
+ }
+
+
+ protected static class AnchoringOutputCollector extends OutputCollector {
+ private Tuple inputTuple;
+
+ AnchoringOutputCollector(IOutputCollector delegate) {
+ super(delegate);
+ }
+
+ void setContext(Tuple inputTuple) {
+ this.inputTuple = inputTuple;
+ }
+
+ @Override
+ public List<Integer> emit(String streamId, List<Object> tuple) {
+ return emit(streamId, inputTuple, tuple);
+ }
+
+ @Override
+ public void emitDirect(int taskId, String streamId, List<Object> tuple) {
+ emitDirect(taskId, streamId, inputTuple, tuple);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/topology/IStatefulBolt.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/IStatefulBolt.java b/storm-core/src/jvm/org/apache/storm/topology/IStatefulBolt.java
new file mode 100644
index 0000000..1c2c5fc
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/topology/IStatefulBolt.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.topology;
+
+import org.apache.storm.state.State;
+
+/**
+ * A bolt abstraction for supporting stateful computation.
+ */
+public interface IStatefulBolt<T extends State> extends IStatefulComponent<T>, IRichBolt {
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/topology/IStatefulComponent.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/IStatefulComponent.java b/storm-core/src/jvm/org/apache/storm/topology/IStatefulComponent.java
new file mode 100644
index 0000000..c2f8789
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/topology/IStatefulComponent.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.topology;
+
+import org.apache.storm.state.State;
+
+/**
+ * <p>
+ * Common methods for stateful components in the topology.
+ * </p>
+ * A stateful component is one that has state (e.g. the result of some computation in a bolt)
+ * and wants the framework to manage its state.
+ */
+public interface IStatefulComponent<T extends State> extends IComponent {
+ /**
+ * This method is invoked by the framework with the previously
+ * saved state of the component. This is invoked after prepare but before
+ * the component starts processing tuples.
+ *
+ * @param state the previously saved state of the component.
+ */
+ void initState(T state);
+
+ /**
+ * This is a hook for the component to perform some actions just before the
+ * framework commits its state.
+ */
+ void preCommit(long txid);
+
+ /**
+ * This is a hook for the component to perform some actions just before the
+ * framework prepares its state.
+ */
+ void prePrepare(long txid);
+
+ /**
+ * This is a hook for the component to perform some actions just before the
+ * framework rolls back the prepared state.
+ */
+ void preRollback();
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java
new file mode 100644
index 0000000..c9c36ee
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/topology/StatefulBoltExecutor.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.topology;
+
+import org.apache.storm.spout.CheckpointSpout;
+import org.apache.storm.state.State;
+import org.apache.storm.state.StateFactory;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.storm.spout.CheckPointState.Action;
+import static org.apache.storm.spout.CheckPointState.Action.COMMIT;
+import static org.apache.storm.spout.CheckPointState.Action.PREPARE;
+import static org.apache.storm.spout.CheckPointState.Action.ROLLBACK;
+import static org.apache.storm.spout.CheckPointState.Action.INITSTATE;
+
+/**
+ * Wraps a {@link IStatefulBolt} and manages the state of the bolt.
+ */
+public class StatefulBoltExecutor<T extends State> extends CheckpointTupleForwarder {
+ private static final Logger LOG = LoggerFactory.getLogger(StatefulBoltExecutor.class);
+ private final IStatefulBolt<T> bolt;
+ private State state;
+ private boolean boltInitialized = false;
+ private List<Tuple> pendingTuples = new ArrayList<>();
+ private List<Tuple> preparedTuples = new ArrayList<>();
+ private List<Tuple> executedTuples = new ArrayList<>();
+
+ public StatefulBoltExecutor(IStatefulBolt<T> bolt) {
+ super(bolt);
+ this.bolt = bolt;
+ }
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ // get the last successfully committed state from state store
+ String namespace = context.getThisComponentId() + "-" + context.getThisTaskId();
+ prepare(stormConf, context, collector, StateFactory.getState(namespace, stormConf, context));
+ }
+
+ // package access for unit tests
+ void prepare(Map stormConf, TopologyContext context, OutputCollector collector, State state) {
+ super.prepare(stormConf, context, collector);
+ this.state = state;
+ }
+
+ @Override
+ protected void handleCheckpoint(Tuple checkpointTuple, Action action, long txid) {
+ LOG.debug("handleCheckPoint with tuple {}, action {}, txid {}", checkpointTuple, action, txid);
+ if (action == PREPARE) {
+ if (boltInitialized) {
+ bolt.prePrepare(txid);
+ state.prepareCommit(txid);
+ preparedTuples.addAll(executedTuples);
+ executedTuples.clear();
+ } else {
+ /*
+ * May be the task restarted in the middle and the state needs be initialized.
+ * Fail fast and trigger recovery.
+ */
+ LOG.debug("Failing checkpointTuple, PREPARE received when bolt state is not initialized.");
+ collector.fail(checkpointTuple);
+ return;
+ }
+ } else if (action == COMMIT) {
+ bolt.preCommit(txid);
+ state.commit(txid);
+ ack(preparedTuples);
+ } else if (action == ROLLBACK) {
+ bolt.preRollback();
+ state.rollback();
+ fail(preparedTuples);
+ fail(executedTuples);
+ } else if (action == INITSTATE) {
+ if (!boltInitialized) {
+ bolt.initState((T) state);
+ boltInitialized = true;
+ LOG.debug("{} pending tuples to process", pendingTuples.size());
+ for (Tuple tuple : pendingTuples) {
+ doExecute(tuple);
+ }
+ pendingTuples.clear();
+ } else {
+ LOG.debug("Bolt state is already initialized, ignoring tuple {}, action {}, txid {}",
+ checkpointTuple, action, txid);
+ }
+ }
+ collector.emit(CheckpointSpout.CHECKPOINT_STREAM_ID, checkpointTuple, new Values(txid, action));
+ collector.ack(checkpointTuple);
+ }
+
+ @Override
+ protected void handleTuple(Tuple input) {
+ if (boltInitialized) {
+ doExecute(input);
+ } else {
+ LOG.debug("Bolt state not initialized, adding tuple {} to pending tuples", input);
+ pendingTuples.add(input);
+ }
+ }
+
+ private void doExecute(Tuple tuple) {
+ collector.setContext(tuple);
+ bolt.execute(tuple);
+ executedTuples.add(tuple);
+ }
+
+ private void ack(List<Tuple> tuples) {
+ if (!tuples.isEmpty()) {
+ LOG.debug("Acking {} tuples", tuples.size());
+ for (Tuple tuple : tuples) {
+ collector.ack(tuple);
+ }
+ tuples.clear();
+ }
+ }
+
+ private void fail(List<Tuple> tuples) {
+ if (!tuples.isEmpty()) {
+ LOG.debug("Failing {} tuples", tuples.size());
+ for (Tuple tuple : tuples) {
+ collector.fail(tuple);
+ }
+ tuples.clear();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java b/storm-core/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java
new file mode 100644
index 0000000..d84d18b
--- /dev/null
+++ b/storm-core/src/jvm/org/apache/storm/topology/base/BaseStatefulBolt.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.topology.base;
+
+import org.apache.storm.state.State;
+import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.topology.IStatefulBolt;
+import org.apache.storm.topology.OutputFieldsDeclarer;
+
+import java.util.Map;
+
+public abstract class BaseStatefulBolt<T extends State> implements IStatefulBolt<T> {
+
+ @Override
+ public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
+ // NOOP
+ }
+
+ @Override
+ public void cleanup() {
+ // NOOP
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ // NOOP
+ }
+
+ @Override
+ public Map<String, Object> getComponentConfiguration() {
+ return null;
+ }
+
+ @Override
+ public void preCommit(long txid) {
+ // NOOP
+ }
+
+ @Override
+ public void prePrepare(long txid) {
+ // NOOP
+ }
+
+ @Override
+ public void preRollback() {
+ // NOOP
+ }
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/test/jvm/backtype/storm/spout/CheckpointSpoutTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/backtype/storm/spout/CheckpointSpoutTest.java b/storm-core/test/jvm/backtype/storm/spout/CheckpointSpoutTest.java
deleted file mode 100644
index 1aa5ab2..0000000
--- a/storm-core/test/jvm/backtype/storm/spout/CheckpointSpoutTest.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package backtype.storm.spout;
-
-import backtype.storm.Config;
-import backtype.storm.state.KeyValueState;
-import backtype.storm.state.State;
-import backtype.storm.state.StateFactory;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.tuple.Values;
-import backtype.storm.utils.Utils;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mockito;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static backtype.storm.spout.CheckPointState.State.COMMITTED;
-import static org.junit.Assert.assertEquals;
-import static backtype.storm.spout.CheckPointState.Action;
-
-/**
- * Unit test for {@link CheckpointSpout}
- */
-public class CheckpointSpoutTest {
- CheckpointSpout spout = new CheckpointSpout();
- TopologyContext mockTopologyContext;
- SpoutOutputCollector mockOutputCollector;
-
- @Before
- public void setUp() throws Exception {
- mockTopologyContext = Mockito.mock(TopologyContext.class);
- Mockito.when(mockTopologyContext.getThisComponentId()).thenReturn("test");
- Mockito.when(mockTopologyContext.getThisTaskId()).thenReturn(1);
- mockOutputCollector = Mockito.mock(SpoutOutputCollector.class);
- }
-
- @Test
- public void testInitState() throws Exception {
- spout.open(new HashMap(), mockTopologyContext, mockOutputCollector);
-
- spout.nextTuple();
- Values expectedTuple = new Values(-1L, Action.INITSTATE);
- ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
- ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
- ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
- Mockito.verify(mockOutputCollector).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(-1L, msgId.getValue());
-
- spout.ack(-1L);
-
- Mockito.verify(mockOutputCollector).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- expectedTuple = new Values(-1L, Action.INITSTATE);
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(-1L, msgId.getValue());
-
- }
-
- @Test
- public void testPrepare() throws Exception {
- spout.open(new HashMap(), mockTopologyContext, mockOutputCollector);
- ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
- ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
- ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
-
- spout.nextTuple();
- spout.ack(-1L);
- spout.nextTuple();
- Mockito.verify(mockOutputCollector, Mockito.times(2)).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- Values expectedTuple = new Values(0L, Action.PREPARE);
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(0L, msgId.getValue());
-
- }
-
- @Test
- public void testPrepareWithFail() throws Exception {
- Map<String, Object> stormConf = new HashMap<>();
- KeyValueState<String, CheckPointState> state =
- (KeyValueState<String, CheckPointState>) StateFactory.getState("__state", stormConf, mockTopologyContext);
- CheckPointState txState = new CheckPointState(-1, COMMITTED);
- state.put("__state", txState);
-
- spout.open(mockTopologyContext, mockOutputCollector, 0, state);
- ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
- ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
- ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
-
- spout.nextTuple();
- spout.ack(-1L);
- Utils.sleep(10);
- spout.nextTuple();
- spout.ack(0L);
- Utils.sleep(10);
- spout.nextTuple();
- spout.ack(0L);
- Utils.sleep(10);
- spout.nextTuple();
- spout.fail(1L);
- Utils.sleep(10);
- spout.nextTuple();
- spout.fail(1L);
- Utils.sleep(10);
- spout.nextTuple();
- spout.ack(1L);
- Utils.sleep(10);
- spout.nextTuple();
- spout.ack(0L);
- Utils.sleep(10);
- spout.nextTuple();
- Mockito.verify(mockOutputCollector, Mockito.times(8)).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- Values expectedTuple = new Values(1L, Action.PREPARE);
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(1L, msgId.getValue());
-
- }
-
- @Test
- public void testCommit() throws Exception {
- Map<String, Object> stormConf = new HashMap();
- stormConf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 0);
- spout.open(stormConf, mockTopologyContext, mockOutputCollector);
- ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
- ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
- ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
-
- spout.nextTuple();
- spout.ack(-1L);
- spout.nextTuple();
- spout.ack(0L);
- Utils.sleep(10);
- spout.nextTuple();
- spout.fail(0L);
- Utils.sleep(10);
- spout.nextTuple();
- Mockito.verify(mockOutputCollector, Mockito.times(4)).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- Values expectedTuple = new Values(0L, Action.COMMIT);
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(0L, msgId.getValue());
-
- }
-
- @Test
- public void testRecoveryRollback() throws Exception {
- Map<String, Object> stormConf = new HashMap();
-
- KeyValueState<String, CheckPointState> state =
- (KeyValueState<String, CheckPointState>) StateFactory.getState("test-1", stormConf, mockTopologyContext);
-
- CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.PREPARING);
- state.put("__state", checkPointState);
- spout.open(mockTopologyContext, mockOutputCollector, 0, state);
- ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
- ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
- ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
-
- spout.nextTuple();
- Mockito.verify(mockOutputCollector, Mockito.times(1)).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- Values expectedTuple = new Values(100L, Action.ROLLBACK);
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(100L, msgId.getValue());
-
- }
-
- @Test
- public void testRecoveryRollbackAck() throws Exception {
- Map<String, Object> stormConf = new HashMap();
-
- KeyValueState<String, CheckPointState> state =
- (KeyValueState<String, CheckPointState>) StateFactory.getState("test-1", stormConf, mockTopologyContext);
-
- CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.PREPARING);
- state.put("__state", checkPointState);
- spout.open(mockTopologyContext, mockOutputCollector, 0, state);
- ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
- ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
- ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
-
- spout.nextTuple();
- spout.ack(100L);
- spout.nextTuple();
- spout.ack(99L);
- spout.nextTuple();
- Mockito.verify(mockOutputCollector, Mockito.times(3)).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- Values expectedTuple = new Values(100L, Action.PREPARE);
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(100L, msgId.getValue());
-
- }
-
- @Test
- public void testRecoveryCommit() throws Exception {
- Map<String, Object> stormConf = new HashMap();
-
- KeyValueState<String, CheckPointState> state =
- (KeyValueState<String, CheckPointState>) StateFactory.getState("test-1", stormConf, mockTopologyContext);
-
- CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.COMMITTING);
- state.put("__state", checkPointState);
- spout.open(mockTopologyContext, mockOutputCollector, 0, state);
- ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
- ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
- ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
-
- spout.nextTuple();
- Mockito.verify(mockOutputCollector, Mockito.times(1)).emit(stream.capture(),
- values.capture(),
- msgId.capture());
-
- Values expectedTuple = new Values(100L, Action.COMMIT);
- assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
- assertEquals(expectedTuple, values.getValue());
- assertEquals(100L, msgId.getValue());
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/test/jvm/backtype/storm/state/InMemoryKeyValueStateTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/backtype/storm/state/InMemoryKeyValueStateTest.java b/storm-core/test/jvm/backtype/storm/state/InMemoryKeyValueStateTest.java
deleted file mode 100644
index 78ace77..0000000
--- a/storm-core/test/jvm/backtype/storm/state/InMemoryKeyValueStateTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package backtype.storm.state;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.*;
-
-/**
- * Unit tests for {@link InMemoryKeyValueState}
- */
-public class InMemoryKeyValueStateTest {
-
- KeyValueState<String, String> keyValueState;
-
- @Before
- public void setUp() {
- keyValueState = new InMemoryKeyValueState<>();
- }
-
- @Test
- public void testPutAndGet() throws Exception {
- keyValueState.put("a", "1");
- keyValueState.put("b", "2");
- assertArrayEquals(new String[]{"1", "2", null}, getValues());
- }
-
- @Test
- public void testPrepareCommitRollback() throws Exception {
- keyValueState.put("a", "1");
- keyValueState.put("b", "2");
- keyValueState.prepareCommit(1);
- keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
- keyValueState.rollback();
- assertArrayEquals(new String[]{null, null, null}, getValues());
- keyValueState.put("a", "1");
- keyValueState.put("b", "2");
- keyValueState.prepareCommit(1);
- keyValueState.commit(1);
- keyValueState.put("c", "3");
- assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
- keyValueState.rollback();
- assertArrayEquals(new String[]{"1", "2", null}, getValues());
- }
-
- private String[] getValues() {
- return new String[]{
- keyValueState.get("a"),
- keyValueState.get("b"),
- keyValueState.get("c")
- };
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/test/jvm/backtype/storm/topology/StatefulBoltExecutorTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/backtype/storm/topology/StatefulBoltExecutorTest.java b/storm-core/test/jvm/backtype/storm/topology/StatefulBoltExecutorTest.java
deleted file mode 100644
index 6f73727..0000000
--- a/storm-core/test/jvm/backtype/storm/topology/StatefulBoltExecutorTest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package backtype.storm.topology;
-
-import backtype.storm.generated.GlobalStreamId;
-import backtype.storm.generated.Grouping;
-import backtype.storm.spout.CheckpointSpout;
-import backtype.storm.state.KeyValueState;
-import backtype.storm.state.State;
-import backtype.storm.task.OutputCollector;
-import backtype.storm.task.TopologyContext;
-import backtype.storm.tuple.Tuple;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static backtype.storm.spout.CheckPointState.Action.*;
-import static backtype.storm.spout.CheckpointSpout.*;
-
-/**
- * Unit tests for {@link StatefulBoltExecutor}
- */
-public class StatefulBoltExecutorTest {
- StatefulBoltExecutor<KeyValueState<String, String>> executor;
- IStatefulBolt<KeyValueState<String, String>> mockBolt;
- TopologyContext mockTopologyContext;
- Tuple mockTuple;
- Tuple mockCheckpointTuple;
- Map<String, Object> mockStormConf = new HashMap<>();
- OutputCollector mockOutputCollector;
- State mockState;
- Map<GlobalStreamId, Grouping> mockGlobalStream;
- Set<GlobalStreamId> mockStreamIds;
- @Before
- public void setUp() throws Exception {
- mockBolt = Mockito.mock(IStatefulBolt.class);
- executor = new StatefulBoltExecutor<>(mockBolt);
- GlobalStreamId mockGlobalStreamId = Mockito.mock(GlobalStreamId.class);
- Mockito.when(mockGlobalStreamId.get_streamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- mockStreamIds = new HashSet<>();
- mockStreamIds.add(mockGlobalStreamId);
- mockTopologyContext = Mockito.mock(TopologyContext.class);
- mockOutputCollector = Mockito.mock(OutputCollector.class);
- mockGlobalStream = Mockito.mock(Map.class);
- mockState = Mockito.mock(State.class);
- Mockito.when(mockTopologyContext.getThisComponentId()).thenReturn("test");
- Mockito.when(mockTopologyContext.getThisTaskId()).thenReturn(1);
- Mockito.when(mockTopologyContext.getThisSources()).thenReturn(mockGlobalStream);
- Mockito.when(mockTopologyContext.getComponentTasks(Mockito.anyString())).thenReturn(Collections.singletonList(1));
- Mockito.when(mockGlobalStream.keySet()).thenReturn(mockStreamIds);
- mockTuple = Mockito.mock(Tuple.class);
- mockCheckpointTuple = Mockito.mock(Tuple.class);
- executor.prepare(mockStormConf, mockTopologyContext, mockOutputCollector, mockState);
- }
-
- @Test
- public void testHandleTupleBeforeInit() throws Exception {
- Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default");
- executor.execute(mockTuple);
- Mockito.verify(mockBolt, Mockito.times(0)).execute(Mockito.any(Tuple.class));
- }
-
- @Test
- public void testHandleTuple() throws Exception {
- Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default");
- executor.execute(mockTuple);
- Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(INITSTATE);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0));
- Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple);
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockBolt, Mockito.times(1)).execute(mockTuple);
- Mockito.verify(mockBolt, Mockito.times(1)).initState(Mockito.any(KeyValueState.class));
- }
-
- @Test
- public void testRollback() throws Exception {
- Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default");
- executor.execute(mockTuple);
- Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0));
- Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple);
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockState, Mockito.times(1)).rollback();
- }
-
- @Test
- public void testCommit() throws Exception {
- Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default");
- executor.execute(mockTuple);
- Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0));
- Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple);
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockBolt, Mockito.times(1)).preCommit(new Long(0));
- Mockito.verify(mockState, Mockito.times(1)).commit(new Long(0));
- }
-
- @Test
- public void testPrepareAndRollbackBeforeInitstate() throws Exception {
- Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default");
- executor.execute(mockTuple);
- Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(PREPARE);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100));
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockOutputCollector, Mockito.times(1)).fail(mockCheckpointTuple);
-
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100));
- Mockito.doNothing().when(mockOutputCollector).ack(mockCheckpointTuple);
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockState, Mockito.times(1)).rollback();
- }
-
- @Test
- public void testCommitBeforeInitstate() throws Exception {
- Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default");
- Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100));
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockCheckpointTuple);
-
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(ROLLBACK);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100));
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockState, Mockito.times(1)).rollback();
- }
-
- @Test
- public void testPrepareAndCommit() throws Exception {
- Mockito.when(mockTuple.getSourceStreamId()).thenReturn("default");
- Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(INITSTATE);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(0));
- executor.execute(mockCheckpointTuple);
-
- executor.execute(mockTuple);
- Mockito.when(mockCheckpointTuple.getSourceStreamId()).thenReturn(CheckpointSpout.CHECKPOINT_STREAM_ID);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(PREPARE);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100));
- executor.execute(mockCheckpointTuple);
- executor.execute(mockTuple);
- Mockito.when(mockCheckpointTuple.getValueByField(CHECKPOINT_FIELD_ACTION)).thenReturn(COMMIT);
- Mockito.when(mockCheckpointTuple.getLongByField(CHECKPOINT_FIELD_TXID)).thenReturn(new Long(100));
- executor.execute(mockCheckpointTuple);
- Mockito.verify(mockState, Mockito.times(1)).commit(new Long(100));
- Mockito.verify(mockBolt, Mockito.times(2)).execute(mockTuple);
- Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockTuple);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java b/storm-core/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java
new file mode 100644
index 0000000..d86f7f3
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/spout/CheckpointSpoutTest.java
@@ -0,0 +1,262 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.spout;
+
+import org.apache.storm.Config;
+import org.apache.storm.state.KeyValueState;
+import org.apache.storm.state.StateFactory;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.tuple.Values;
+import org.apache.storm.utils.Utils;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.storm.spout.CheckPointState.State.COMMITTED;
+import static org.junit.Assert.assertEquals;
+import static org.apache.storm.spout.CheckPointState.Action;
+
+/**
+ * Unit test for {@link CheckpointSpout}
+ */
+public class CheckpointSpoutTest {
+ CheckpointSpout spout = new CheckpointSpout();
+ TopologyContext mockTopologyContext;
+ SpoutOutputCollector mockOutputCollector;
+
+ @Before
+ public void setUp() throws Exception {
+ mockTopologyContext = Mockito.mock(TopologyContext.class);
+ Mockito.when(mockTopologyContext.getThisComponentId()).thenReturn("test");
+ Mockito.when(mockTopologyContext.getThisTaskId()).thenReturn(1);
+ mockOutputCollector = Mockito.mock(SpoutOutputCollector.class);
+ }
+
+ @Test
+ public void testInitState() throws Exception {
+ spout.open(new HashMap(), mockTopologyContext, mockOutputCollector);
+
+ spout.nextTuple();
+ Values expectedTuple = new Values(-1L, Action.INITSTATE);
+ ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
+ ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
+ Mockito.verify(mockOutputCollector).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(-1L, msgId.getValue());
+
+ spout.ack(-1L);
+
+ Mockito.verify(mockOutputCollector).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ expectedTuple = new Values(-1L, Action.INITSTATE);
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(-1L, msgId.getValue());
+
+ }
+
+ @Test
+ public void testPrepare() throws Exception {
+ spout.open(new HashMap(), mockTopologyContext, mockOutputCollector);
+ ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
+ ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
+
+ spout.nextTuple();
+ spout.ack(-1L);
+ spout.nextTuple();
+ Mockito.verify(mockOutputCollector, Mockito.times(2)).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ Values expectedTuple = new Values(0L, Action.PREPARE);
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(0L, msgId.getValue());
+
+ }
+
+ @Test
+ public void testPrepareWithFail() throws Exception {
+ Map<String, Object> stormConf = new HashMap<>();
+ KeyValueState<String, CheckPointState> state =
+ (KeyValueState<String, CheckPointState>) StateFactory.getState("__state", stormConf, mockTopologyContext);
+ CheckPointState txState = new CheckPointState(-1, COMMITTED);
+ state.put("__state", txState);
+
+ spout.open(mockTopologyContext, mockOutputCollector, 0, state);
+ ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
+ ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
+
+ spout.nextTuple();
+ spout.ack(-1L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ spout.ack(0L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ spout.ack(0L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ spout.fail(1L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ spout.fail(1L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ spout.ack(1L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ spout.ack(0L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ Mockito.verify(mockOutputCollector, Mockito.times(8)).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ Values expectedTuple = new Values(1L, Action.PREPARE);
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(1L, msgId.getValue());
+
+ }
+
+ @Test
+ public void testCommit() throws Exception {
+ Map<String, Object> stormConf = new HashMap();
+ stormConf.put(Config.TOPOLOGY_STATE_CHECKPOINT_INTERVAL, 0);
+ spout.open(stormConf, mockTopologyContext, mockOutputCollector);
+ ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
+ ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
+
+ spout.nextTuple();
+ spout.ack(-1L);
+ spout.nextTuple();
+ spout.ack(0L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ spout.fail(0L);
+ Utils.sleep(10);
+ spout.nextTuple();
+ Mockito.verify(mockOutputCollector, Mockito.times(4)).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ Values expectedTuple = new Values(0L, Action.COMMIT);
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(0L, msgId.getValue());
+
+ }
+
+ @Test
+ public void testRecoveryRollback() throws Exception {
+ Map<String, Object> stormConf = new HashMap();
+
+ KeyValueState<String, CheckPointState> state =
+ (KeyValueState<String, CheckPointState>) StateFactory.getState("test-1", stormConf, mockTopologyContext);
+
+ CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.PREPARING);
+ state.put("__state", checkPointState);
+ spout.open(mockTopologyContext, mockOutputCollector, 0, state);
+ ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
+ ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
+
+ spout.nextTuple();
+ Mockito.verify(mockOutputCollector, Mockito.times(1)).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ Values expectedTuple = new Values(100L, Action.ROLLBACK);
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(100L, msgId.getValue());
+
+ }
+
+ @Test
+ public void testRecoveryRollbackAck() throws Exception {
+ Map<String, Object> stormConf = new HashMap();
+
+ KeyValueState<String, CheckPointState> state =
+ (KeyValueState<String, CheckPointState>) StateFactory.getState("test-1", stormConf, mockTopologyContext);
+
+ CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.PREPARING);
+ state.put("__state", checkPointState);
+ spout.open(mockTopologyContext, mockOutputCollector, 0, state);
+ ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
+ ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
+
+ spout.nextTuple();
+ spout.ack(100L);
+ spout.nextTuple();
+ spout.ack(99L);
+ spout.nextTuple();
+ Mockito.verify(mockOutputCollector, Mockito.times(3)).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ Values expectedTuple = new Values(100L, Action.PREPARE);
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(100L, msgId.getValue());
+
+ }
+
+ @Test
+ public void testRecoveryCommit() throws Exception {
+ Map<String, Object> stormConf = new HashMap();
+
+ KeyValueState<String, CheckPointState> state =
+ (KeyValueState<String, CheckPointState>) StateFactory.getState("test-1", stormConf, mockTopologyContext);
+
+ CheckPointState checkPointState = new CheckPointState(100, CheckPointState.State.COMMITTING);
+ state.put("__state", checkPointState);
+ spout.open(mockTopologyContext, mockOutputCollector, 0, state);
+ ArgumentCaptor<String> stream = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<Values> values = ArgumentCaptor.forClass(Values.class);
+ ArgumentCaptor<Object> msgId = ArgumentCaptor.forClass(Object.class);
+
+ spout.nextTuple();
+ Mockito.verify(mockOutputCollector, Mockito.times(1)).emit(stream.capture(),
+ values.capture(),
+ msgId.capture());
+
+ Values expectedTuple = new Values(100L, Action.COMMIT);
+ assertEquals(CheckpointSpout.CHECKPOINT_STREAM_ID, stream.getValue());
+ assertEquals(expectedTuple, values.getValue());
+ assertEquals(100L, msgId.getValue());
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/bfc5ffd5/storm-core/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java b/storm-core/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java
new file mode 100644
index 0000000..361865b
--- /dev/null
+++ b/storm-core/test/jvm/org/apache/storm/state/InMemoryKeyValueStateTest.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.state;
+
+import org.apache.storm.state.InMemoryKeyValueState;
+import org.apache.storm.state.KeyValueState;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unit tests for {@link InMemoryKeyValueState}
+ */
+public class InMemoryKeyValueStateTest {
+
+ KeyValueState<String, String> keyValueState;
+
+ @Before
+ public void setUp() {
+ keyValueState = new InMemoryKeyValueState<>();
+ }
+
+ @Test
+ public void testPutAndGet() throws Exception {
+ keyValueState.put("a", "1");
+ keyValueState.put("b", "2");
+ assertArrayEquals(new String[]{"1", "2", null}, getValues());
+ }
+
+ @Test
+ public void testPrepareCommitRollback() throws Exception {
+ keyValueState.put("a", "1");
+ keyValueState.put("b", "2");
+ keyValueState.prepareCommit(1);
+ keyValueState.put("c", "3");
+ assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ keyValueState.rollback();
+ assertArrayEquals(new String[]{null, null, null}, getValues());
+ keyValueState.put("a", "1");
+ keyValueState.put("b", "2");
+ keyValueState.prepareCommit(1);
+ keyValueState.commit(1);
+ keyValueState.put("c", "3");
+ assertArrayEquals(new String[]{"1", "2", "3"}, getValues());
+ keyValueState.rollback();
+ assertArrayEquals(new String[]{"1", "2", null}, getValues());
+ }
+
+ private String[] getValues() {
+ return new String[]{
+ keyValueState.get("a"),
+ keyValueState.get("b"),
+ keyValueState.get("c")
+ };
+ }
+}
\ No newline at end of file