You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nl...@apache.org on 2018/07/27 21:31:56 UTC
[incubator-heron] branch master updated: spill state to local disk
when configured (#2967)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 5c32f4d spill state to local disk when configured (#2967)
5c32f4d is described below
commit 5c32f4dedd5b8ebd3daebb5d2f69732c070d4c7f
Author: Neng Lu <fr...@gmail.com>
AuthorDate: Fri Jul 27 14:31:53 2018 -0700
spill state to local disk when configured (#2967)
* spill state to local disk when configured
* start ckptmgr-client after receiving pplan
* separate instances and ckptmgr spilled location
* clean states by owner
* bail out if spill state failed
* typo
---
.../api/StatefulTumblingWindowTopology.java | 4 +-
.../api/src/java/org/apache/heron/api/Config.java | 30 ++++++++
.../java/org/apache/heron/api/HeronTopology.java | 6 ++
heron/ckptmgr/src/java/BUILD | 8 ++-
.../heron/ckptmgr/CheckpointManagerServer.java | 79 ++++++++++++++++++++--
.../heron/ckptmgr/CheckpointManagerServerTest.java | 14 ++++
.../heron/instance/AbstractOutputCollector.java | 6 +-
.../heron/instance/OutgoingTupleCollection.java | 36 +++++++---
.../src/java/org/apache/heron/instance/Slave.java | 10 +++
.../apache/heron/instance/bolt/BoltInstance.java | 31 +++++++--
.../apache/heron/instance/spout/SpoutInstance.java | 22 +++++-
heron/proto/ckptmgr.proto | 7 +-
heron/stmgr/src/cpp/manager/ckptmgr-client.cpp | 6 ++
heron/stmgr/src/cpp/manager/ckptmgr-client.h | 6 +-
heron/stmgr/src/cpp/manager/stmgr.cpp | 38 +++++------
15 files changed, 250 insertions(+), 53 deletions(-)
diff --git a/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java b/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
index b39e3ff..62c8185 100644
--- a/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
+++ b/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
@@ -155,7 +155,8 @@ public final class StatefulTumblingWindowTopology {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("integer", new IntegerSpout(), 1);
WindowSumBolt windowSumBolt = new WindowSumBolt();
- windowSumBolt.withTumblingWindow(Duration.ofSeconds(10));
+ windowSumBolt.withTumblingWindow(Duration.ofMinutes(10));
+
builder.setBolt("sumbolt", windowSumBolt, 1).shuffleGrouping("integer");
builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("sumbolt");
Config conf = new Config();
@@ -172,6 +173,7 @@ public final class StatefulTumblingWindowTopology {
conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
conf.setTopologyStatefulCheckpointIntervalSecs(20);
conf.setMaxSpoutPending(1000);
+ conf.setMessageTimeoutSecs(1500);
if (args != null && args.length > 0) {
topoName = args[0];
diff --git a/heron/api/src/java/org/apache/heron/api/Config.java b/heron/api/src/java/org/apache/heron/api/Config.java
index 2e22731..d91c8a0 100644
--- a/heron/api/src/java/org/apache/heron/api/Config.java
+++ b/heron/api/src/java/org/apache/heron/api/Config.java
@@ -223,6 +223,16 @@ public class Config extends HashMap<String, Object> {
public static final String TOPOLOGY_STATEFUL_CKPTMGR_RAM =
"topology.stateful.checkpointmanager.ram";
/**
+ * Whether spill the state to disk for transferring
+ */
+ public static final String TOPOLOGY_STATEFUL_SPILL_STATE =
+ "topology.stateful.spill.state";
+ /**
+ * The local disk location where the state is spilled
+ */
+ public static final String TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION =
+ "topology.stateful.spill.state.location";
+ /**
* Stream Manager RAM requirement
*/
public static final String TOPOLOGY_STMGR_RAM = "topology.stmgr.ram";
@@ -333,6 +343,8 @@ public class Config extends HashMap<String, Object> {
apiVars.add(TOPOLOGY_STATEFUL_START_CLEAN);
apiVars.add(TOPOLOGY_STATEFUL_CHECKPOINT_INTERVAL_SECONDS);
apiVars.add(TOPOLOGY_STATEFUL_CKPTMGR_RAM);
+ apiVars.add(TOPOLOGY_STATEFUL_SPILL_STATE);
+ apiVars.add(TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION);
apiVars.add(TOPOLOGY_STMGR_RAM);
apiVars.add(TOPOLOGY_METRICSMGR_RAM);
apiVars.add(TOPOLOGY_RELIABILITY_MODE);
@@ -593,6 +605,16 @@ public class Config extends HashMap<String, Object> {
conf.put(Config.TOPOLOGY_STATEFUL_CKPTMGR_RAM, ramInBytes.asBytes());
}
+ public static void setTopologyStatefulSpillState(Map<String, Object> conf,
+ String spillState) {
+ conf.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE, spillState);
+ }
+
+ public static void setTopologyStatefulSpillStateLocation(Map<String, Object> conf,
+ String location) {
+ conf.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION, location);
+ }
+
public static void setStreamManagerRam(Map<String, Object> conf,
ByteAmount ramInBytes) {
conf.put(Config.TOPOLOGY_STMGR_RAM, ramInBytes.asBytes());
@@ -773,6 +795,14 @@ public class Config extends HashMap<String, Object> {
setCheckpointManagerRam(this, ramInBytes);
}
+ public void setTopologyStatefulSpillState(String spillState) {
+ setTopologyStatefulSpillState(this, spillState);
+ }
+
+ public void setTopologyStatefulSpillStateLocation(String location) {
+ setTopologyStatefulSpillStateLocation(this, location);
+ }
+
public void setStreamManagerRam(ByteAmount ramInBytes) {
setStreamManagerRam(this, ramInBytes);
}
diff --git a/heron/api/src/java/org/apache/heron/api/HeronTopology.java b/heron/api/src/java/org/apache/heron/api/HeronTopology.java
index e0aaeff..a6263f7 100644
--- a/heron/api/src/java/org/apache/heron/api/HeronTopology.java
+++ b/heron/api/src/java/org/apache/heron/api/HeronTopology.java
@@ -62,6 +62,12 @@ public class HeronTopology {
if (!userConfig.containsKey(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS)) {
userConfig.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, "true");
}
+ if (!userConfig.containsKey(Config.TOPOLOGY_STATEFUL_SPILL_STATE)) {
+ userConfig.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE, "false");
+ if (!userConfig.containsKey(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION)) {
+ userConfig.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION, "./spilled-state/");
+ }
+ }
}
public TopologyAPI.Topology getTopology() {
diff --git a/heron/ckptmgr/src/java/BUILD b/heron/ckptmgr/src/java/BUILD
index 5a7799b..e139b13 100644
--- a/heron/ckptmgr/src/java/BUILD
+++ b/heron/ckptmgr/src/java/BUILD
@@ -9,13 +9,17 @@ java_library(
exclude = ["**/CheckpointManager.java"],
),
deps = [
- "//heron/spi/src/java:statefulstorage-spi-java",
+ "//heron/api/src/java:api-java-low-level",
"//heron/common/src/java:basics-java",
"//heron/common/src/java:config-java",
"//heron/common/src/java:network-java",
"//heron/common/src/java:utils-java",
"//heron/proto:proto_common_java",
"//heron/proto:proto_ckptmgr_java",
+ "//heron/proto:proto_physical_plan_java",
+ "//heron/proto:proto_stmgr_java",
+ "//heron/proto:proto_topology_java",
+ "//heron/spi/src/java:statefulstorage-spi-java",
"@com_google_protobuf//:protobuf_java",
],
)
@@ -25,13 +29,13 @@ java_binary(
srcs = glob(["**/CheckpointManager.java"]),
deps = [
":ckptmgr-java",
- "//heron/spi/src/java:statefulstorage-spi-java",
"//heron/api/src/java:api-java-low-level",
"//heron/common/src/java:basics-java",
"//heron/common/src/java:config-java",
"//heron/common/src/java:network-java",
"//heron/common/src/java:utils-java",
"//heron/proto:proto_ckptmgr_java",
+ "//heron/spi/src/java:statefulstorage-spi-java",
"@commons_cli_commons_cli//jar",
],
)
diff --git a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
index 4b4654a..f6c3416 100644
--- a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
+++ b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
@@ -21,18 +21,24 @@ package org.apache.heron.ckptmgr;
import java.io.IOException;
import java.nio.channels.SocketChannel;
+import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.google.protobuf.ByteString;
import com.google.protobuf.Message;
+import org.apache.heron.api.Config;
+import org.apache.heron.api.generated.TopologyAPI;
+import org.apache.heron.api.utils.TopologyUtils;
+import org.apache.heron.common.basics.FileUtils;
import org.apache.heron.common.basics.NIOLooper;
import org.apache.heron.common.network.HeronServer;
import org.apache.heron.common.network.HeronSocketOptions;
import org.apache.heron.common.network.REQID;
import org.apache.heron.proto.ckptmgr.CheckpointManager;
import org.apache.heron.proto.system.Common;
+import org.apache.heron.proto.system.PhysicalPlans;
import org.apache.heron.spi.statefulstorage.Checkpoint;
import org.apache.heron.spi.statefulstorage.CheckpointInfo;
import org.apache.heron.spi.statefulstorage.IStatefulStorage;
@@ -45,6 +51,8 @@ public class CheckpointManagerServer extends HeronServer {
private final String topologyId;
private final String checkpointMgrId;
private final IStatefulStorage statefulStorage;
+ private boolean spillState;
+ private String spillStateLocation;
private SocketChannel connection;
@@ -63,6 +71,8 @@ public class CheckpointManagerServer extends HeronServer {
this.topologyId = topologyId;
this.checkpointMgrId = checkpointMgrId;
this.statefulStorage = statefulStorage;
+ this.spillState = false;
+ this.spillStateLocation = "";
this.connection = null;
@@ -108,6 +118,10 @@ public class CheckpointManagerServer extends HeronServer {
}
}
+ @Override
+ public void onMessage(SocketChannel channel, Message message) {
+ }
+
protected void handleCleanStatefulCheckpointRequest(
REQID rid,
SocketChannel channel,
@@ -178,6 +192,7 @@ public class CheckpointManagerServer extends HeronServer {
) {
LOG.info(String.format("Got a StMgr register request from %s running on host:port %s",
request.getStmgrId(), channel.socket().getRemoteSocketAddress()));
+ handlePhysicalPlan(request.getPhysicalPlan());
CheckpointManager.RegisterStMgrResponse.Builder responseBuilder =
CheckpointManager.RegisterStMgrResponse.newBuilder();
@@ -210,8 +225,15 @@ public class CheckpointManagerServer extends HeronServer {
) {
CheckpointInfo info =
new CheckpointInfo(request.getCheckpoint().getCheckpointId(),
- request.getInstance());
- Checkpoint checkpoint = new Checkpoint(request.getCheckpoint());
+ request.getInstance());
+
+ Checkpoint checkpoint;
+ if (request.getCheckpoint().hasStateLocation()) {
+ checkpoint = loadCheckpoint(request.getInstance(), request.getCheckpoint().getCheckpointId(),
+ request.getCheckpoint().getStateLocation());
+ } else {
+ checkpoint = new Checkpoint(request.getCheckpoint());
+ }
LOG.info(String.format("Got a save checkpoint request for checkpointId %s "
+ " component %s instanceId %s on connection %s",
@@ -246,6 +268,18 @@ public class CheckpointManagerServer extends HeronServer {
sendResponse(rid, channel, responseBuilder.build());
}
+ private Checkpoint loadCheckpoint(PhysicalPlans.Instance instanceInfo,
+ String checkpointId, String localStateLocation) {
+ LOG.info("fetch state from: " + localStateLocation);
+ CheckpointManager.InstanceStateCheckpoint checkpoint =
+ CheckpointManager.InstanceStateCheckpoint.newBuilder()
+ .setCheckpointId(checkpointId)
+ .setState(ByteString.copyFrom(FileUtils.readFromFile(localStateLocation)))
+ .build();
+
+ return new Checkpoint(checkpoint);
+ }
+
protected void handleGetInstanceStateRequest(
REQID rid,
SocketChannel channel,
@@ -283,7 +317,28 @@ public class CheckpointManagerServer extends HeronServer {
info.getComponent(),
info.getInstanceId()));
// Set the checkpoint-state in response
- responseBuilder.setCheckpoint(checkpoint.getCheckpoint());
+ if (spillState) {
+ CheckpointManager.InstanceStateCheckpoint ckpt = checkpoint.getCheckpoint();
+ String checkpointId = ckpt.getCheckpointId();
+
+ // clean any possible existing states
+ FileUtils.cleanDir(spillStateLocation);
+
+ // spill state to local disk
+ String stateLocation = spillStateLocation + checkpointId + "-" + UUID.randomUUID();
+ if (!FileUtils.writeToFile(stateLocation, ckpt.getState().toByteArray(), true)) {
+ throw new RuntimeException("failed to spill state. Bailing out...");
+ }
+ LOG.info("spilled state to: " + stateLocation);
+ ckpt = CheckpointManager.InstanceStateCheckpoint.newBuilder()
+ .setStateLocation(stateLocation)
+ .setCheckpointId(checkpointId)
+ .build();
+
+ responseBuilder.setCheckpoint(ckpt);
+ } else {
+ responseBuilder.setCheckpoint(checkpoint.getCheckpoint());
+ }
} catch (StatefulStorageException e) {
errorMessage = String.format("Get checkpoint not successful for checkpointId %s "
+ "component %s instanceId %d",
@@ -301,9 +356,21 @@ public class CheckpointManagerServer extends HeronServer {
sendResponse(rid, channel, responseBuilder.build());
}
- @Override
- public void onMessage(SocketChannel channel, Message message) {
-
+ private void handlePhysicalPlan(PhysicalPlans.PhysicalPlan pplan) {
+ TopologyAPI.Config config = pplan.getTopology().getTopologyConfig();
+ this.spillState = Boolean.parseBoolean(TopologyUtils.getConfigWithDefault(config.getKvsList(),
+ Config.TOPOLOGY_STATEFUL_SPILL_STATE, "false"));
+ this.spillStateLocation = String.format("%s/%s/",
+ String.valueOf(TopologyUtils.getConfigWithDefault(config.getKvsList(),
+ Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION, "")),
+ "ckptmgr");
+ if (FileUtils.isDirectoryExists(spillStateLocation)) {
+ FileUtils.cleanDir(spillStateLocation);
+ } else {
+ FileUtils.createDirectory(spillStateLocation);
+ }
+ LOG.info("spill state: " + spillState
+ + ". set spilled state location: " + spillStateLocation);
}
@Override
diff --git a/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java b/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java
index c6bffc7..34eba63 100644
--- a/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java
+++ b/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java
@@ -30,6 +30,7 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.heron.api.generated.TopologyAPI;
import org.apache.heron.common.basics.NIOLooper;
import org.apache.heron.common.basics.SysUtils;
import org.apache.heron.common.network.HeronClient;
@@ -79,6 +80,8 @@ public class CheckpointManagerServerTest {
final int COMPONENT_INDEX = 1;
final String COMPONENT_NAME = "component_name";
final byte[] BYTES = "checkpoint manager server test bytes".getBytes();
+ final String TOPO_ID = "topo_id";
+ final String TOPO_NAME = "topo_name";
PhysicalPlans.InstanceInfo info = PhysicalPlans.InstanceInfo.newBuilder()
.setTaskId(TASK_ID)
@@ -86,6 +89,16 @@ public class CheckpointManagerServerTest {
.setComponentName(COMPONENT_NAME)
.build();
+ TopologyAPI.Topology topology = TopologyAPI.Topology.newBuilder()
+ .setId(TOPO_ID)
+ .setName(TOPO_NAME)
+ .setState(TopologyAPI.TopologyState.RUNNING)
+ .build();
+
+ PhysicalPlans.PhysicalPlan pplan = PhysicalPlans.PhysicalPlan.newBuilder()
+ .setTopology(topology)
+ .build();
+
instance = PhysicalPlans.Instance.newBuilder()
.setInstanceId(INSTANCE_ID)
.setStmgrId(STMGR_ID)
@@ -121,6 +134,7 @@ public class CheckpointManagerServerTest {
.setTopologyId(TOPOLOGY_ID)
.setStmgrId(STMGR_ID)
.setTopologyName(TOPOLOGY_NAME)
+ .setPhysicalPlan(pplan)
.build();
registerTMasterRequest = CheckpointManager.RegisterTMasterRequest.newBuilder()
diff --git a/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java b/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java
index 8d5f1fa..08afa61 100644
--- a/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java
+++ b/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java
@@ -114,8 +114,10 @@ public class AbstractOutputCollector {
// Flush the states
public void sendOutState(State<Serializable, Serializable> state,
- String checkpointId) {
- outputter.sendOutState(state, checkpointId);
+ String checkpointId,
+ boolean spillState,
+ String location) {
+ outputter.sendOutState(state, checkpointId, spillState, location);
}
// Clean the internal state of BoltOutputCollectorImpl
diff --git a/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java b/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java
index 7c28290..862c300 100644
--- a/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java
+++ b/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java
@@ -20,6 +20,7 @@
package org.apache.heron.instance;
import java.io.Serializable;
+import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
@@ -31,6 +32,7 @@ import org.apache.heron.api.serializer.IPluggableSerializer;
import org.apache.heron.api.state.State;
import org.apache.heron.common.basics.ByteAmount;
import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
import org.apache.heron.common.basics.SingletonRegistry;
import org.apache.heron.common.config.SystemConfig;
import org.apache.heron.common.utils.misc.PhysicalPlanHelper;
@@ -102,30 +104,44 @@ public class OutgoingTupleCollection {
}
/**
- * Send out the instance's state with corresponding checkpointId
+ * Send out the instance's state with corresponding checkpointId. If spillState is True,
+ * the actual state is spill to disk and only the state location is sent out.
* @param state instance's state
* @param checkpointId the checkpointId
+ * @param spillState spill the state to local disk if true
+ * @param location the location where state is spilled
*/
public void sendOutState(State<Serializable, Serializable> state,
- String checkpointId) {
+ String checkpointId,
+ boolean spillState,
+ String location) {
lock.lock();
try {
// flush all the current data before sending the state
flushRemaining();
- // Serialize the state
+ // serialize the state
byte[] serializedState = serializer.serialize(state);
- // Construct the instance state checkpoint
- CheckpointManager.InstanceStateCheckpoint instanceState =
- CheckpointManager.InstanceStateCheckpoint.newBuilder()
- .setCheckpointId(checkpointId)
- .setState(ByteString.copyFrom(serializedState))
- .build();
+ CheckpointManager.InstanceStateCheckpoint.Builder instanceStateBuilder =
+ CheckpointManager.InstanceStateCheckpoint.newBuilder();
+ instanceStateBuilder.setCheckpointId(checkpointId);
+
+ if (spillState) {
+ FileUtils.cleanDir(location);
+
+ String stateLocation = location + checkpointId + "-" + UUID.randomUUID();
+ if (!FileUtils.writeToFile(stateLocation, serializedState, true)) {
+ throw new RuntimeException("failed to spill state. Bailing out...");
+ }
+ instanceStateBuilder.setStateLocation(stateLocation);
+ } else {
+ instanceStateBuilder.setState(ByteString.copyFrom(serializedState));
+ }
CheckpointManager.StoreInstanceStateCheckpoint storeRequest =
CheckpointManager.StoreInstanceStateCheckpoint.newBuilder()
- .setState(instanceState)
+ .setState(instanceStateBuilder.build())
.build();
// Put the checkpoint to out stream queue
diff --git a/heron/instance/src/java/org/apache/heron/instance/Slave.java b/heron/instance/src/java/org/apache/heron/instance/Slave.java
index 7d8e1f8..5b85fc6 100644
--- a/heron/instance/src/java/org/apache/heron/instance/Slave.java
+++ b/heron/instance/src/java/org/apache/heron/instance/Slave.java
@@ -32,6 +32,7 @@ import org.apache.heron.api.serializer.IPluggableSerializer;
import org.apache.heron.api.state.HashMapState;
import org.apache.heron.api.state.State;
import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
import org.apache.heron.common.basics.SingletonRegistry;
import org.apache.heron.common.basics.SlaveLooper;
import org.apache.heron.common.config.SystemConfig;
@@ -304,6 +305,7 @@ public class Slave implements Runnable, AutoCloseable {
instanceState.clear();
instanceState = null;
}
+
if (request.getState().hasState() && !request.getState().getState().isEmpty()) {
@SuppressWarnings("unchecked")
State<Serializable, Serializable> stateToRestore =
@@ -311,6 +313,14 @@ public class Slave implements Runnable, AutoCloseable {
request.getState().getState().toByteArray());
instanceState = stateToRestore;
+ } else if (request.getState().hasStateLocation()) {
+ String stateLocation = request.getState().getStateLocation();
+ byte[] rawState = FileUtils.readFromFile(stateLocation);
+
+ @SuppressWarnings("unchecked")
+ State<Serializable, Serializable> stateToRestore =
+ (State<Serializable, Serializable>) serializer.deserialize(rawState);
+ instanceState = stateToRestore;
} else {
LOG.info("The restore request does not have an actual state");
}
diff --git a/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java b/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
index cd49dac..f58e181 100644
--- a/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
+++ b/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
@@ -40,6 +40,7 @@ import org.apache.heron.api.topology.IStatefulComponent;
import org.apache.heron.api.topology.IUpdatable;
import org.apache.heron.api.utils.Utils;
import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
import org.apache.heron.common.basics.SingletonRegistry;
import org.apache.heron.common.basics.SlaveLooper;
import org.apache.heron.common.basics.TypeUtils;
@@ -68,13 +69,21 @@ public class BoltInstance implements IInstance {
private final Communicator<Message> streamInQueue;
private final boolean isTopologyStateful;
+ private final boolean spillState;
+ private final String spillStateLocation;
private State<Serializable, Serializable> instanceState;
+ // The reference to topology's config
+ private final Map<String, Object> config;
+
private final SlaveLooper looper;
private final SystemConfig systemConfig;
+ /**
+ * Construct a BoltInstance basing on given arguments
+ */
public BoltInstance(PhysicalPlanHelper helper,
Communicator<Message> streamInQueue,
Communicator<Message> streamOutQueue,
@@ -89,12 +98,18 @@ public class BoltInstance implements IInstance {
this.systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(
SystemConfig.HERON_SYSTEM_CONFIG);
- Map<String, Object> config = helper.getTopologyContext().getTopologyConfig();
+ this.config = helper.getTopologyContext().getTopologyConfig();
this.isTopologyStateful = String.valueOf(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE)
.equals(config.get(Config.TOPOLOGY_RELIABILITY_MODE));
-
LOG.info("Is this topology stateful: " + isTopologyStateful);
+ this.spillState =
+ Boolean.parseBoolean((String) config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE));
+
+ this.spillStateLocation = String.format("%s/%s/",
+ String.valueOf(config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION)),
+ helper.getMyInstanceId());
+
if (helper.getMyBolt() == null) {
throw new RuntimeException("HeronBoltInstance has no bolt in physical plan.");
}
@@ -143,7 +158,6 @@ public class BoltInstance implements IInstance {
if (!isTopologyStateful) {
throw new RuntimeException("Could not save a non-stateful topology's state");
}
-
// need to synchronize with other OutgoingTupleCollection operations
// so that topology emit, ack, fail are thread safe
collector.lock.lock();
@@ -152,8 +166,7 @@ public class BoltInstance implements IInstance {
if (bolt instanceof IStatefulComponent) {
((IStatefulComponent) bolt).preSave(checkpointId);
}
-
- collector.sendOutState(instanceState, checkpointId);
+ collector.sendOutState(instanceState, checkpointId, spillState, spillStateLocation);
} finally {
collector.lock.unlock();
}
@@ -174,6 +187,14 @@ public class BoltInstance implements IInstance {
if (bolt instanceof IStatefulComponent) {
this.instanceState = state;
((IStatefulComponent<Serializable, Serializable>) bolt).initState(instanceState);
+
+ if (spillState) {
+ if (FileUtils.isDirectoryExists(spillStateLocation)) {
+ FileUtils.cleanDir(spillStateLocation);
+ } else {
+ FileUtils.createDirectory(spillStateLocation);
+ }
+ }
}
// Delegate
diff --git a/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java b/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
index f7c077a..5fc1fdb 100644
--- a/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
+++ b/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
@@ -40,6 +40,7 @@ import org.apache.heron.api.topology.IUpdatable;
import org.apache.heron.api.utils.Utils;
import org.apache.heron.common.basics.ByteAmount;
import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
import org.apache.heron.common.basics.SingletonRegistry;
import org.apache.heron.common.basics.SlaveLooper;
import org.apache.heron.common.basics.TypeUtils;
@@ -67,6 +68,9 @@ public class SpoutInstance implements IInstance {
private final boolean enableMessageTimeouts;
private final boolean isTopologyStateful;
+ private final boolean spillState;
+ private final String spillStateLocation;
+
private State<Serializable, Serializable> instanceState;
private final SlaveLooper looper;
@@ -99,6 +103,13 @@ public class SpoutInstance implements IInstance {
this.isTopologyStateful = String.valueOf(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE)
.equals(config.get(Config.TOPOLOGY_RELIABILITY_MODE));
+ this.spillState =
+ Boolean.parseBoolean((String) config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE));
+
+ this.spillStateLocation = String.format("%s/%s/",
+ String.valueOf(config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION)),
+ helper.getMyInstanceId());
+
LOG.info("Is this topology stateful: " + isTopologyStateful);
if (helper.getMySpout() == null) {
@@ -160,8 +171,7 @@ public class SpoutInstance implements IInstance {
if (spout instanceof IStatefulComponent) {
((IStatefulComponent) spout).preSave(checkpointId);
}
-
- collector.sendOutState(instanceState, checkpointId);
+ collector.sendOutState(instanceState, checkpointId, spillState, spillStateLocation);
} finally {
collector.lock.unlock();
}
@@ -182,6 +192,14 @@ public class SpoutInstance implements IInstance {
if (spout instanceof IStatefulComponent) {
this.instanceState = state;
((IStatefulComponent<Serializable, Serializable>) spout).initState(instanceState);
+
+ if (spillState) {
+ if (FileUtils.isDirectoryExists(spillStateLocation)) {
+ FileUtils.cleanDir(spillStateLocation);
+ } else {
+ FileUtils.createDirectory(spillStateLocation);
+ }
+ }
}
spout.open(
diff --git a/heron/proto/ckptmgr.proto b/heron/proto/ckptmgr.proto
index 68935cc..f21e6ed 100644
--- a/heron/proto/ckptmgr.proto
+++ b/heron/proto/ckptmgr.proto
@@ -180,10 +180,12 @@ message CleanStatefulCheckpointResponse {
*/
// This message encapsulates the info associated with
-// state of an instance/partition
+// state of an instance/partition. only one of the state_location
+// and state should be set
message InstanceStateCheckpoint {
required string checkpoint_id = 1;
- required bytes state = 2;
+ optional bytes state = 2;
+ optional string state_location = 3;
}
// This message encapsulates the info associated with
@@ -200,6 +202,7 @@ message RegisterStMgrRequest {
required string topology_name = 1;
required string topology_id = 2;
required string stmgr_id = 3;
+ required heron.proto.system.PhysicalPlan physical_plan = 4;
}
// This is the message that checkpoint manager
diff --git a/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp b/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp
index c6e6a04..a41bf9b 100644
--- a/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp
+++ b/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp
@@ -42,6 +42,7 @@ CkptMgrClient::CkptMgrClient(EventLoop* eventloop, const NetworkOptions& _option
ckptmgr_id_(_ckptmgr_id),
stmgr_id_(_stmgr_id),
quit_(false),
+ pplan_(nullptr),
ckpt_saved_watcher_(_ckpt_saved_watcher),
ckpt_get_watcher_(_ckpt_get_watcher),
register_watcher_(_register_watcher) {
@@ -144,6 +145,7 @@ void CkptMgrClient::SendRegisterRequest() {
request->set_topology_name(topology_name_);
request->set_topology_id(topology_id_);
request->set_stmgr_id(stmgr_id_);
+ request->mutable_physical_plan()->CopyFrom(*pplan_);
SendRequest(request, NULL);
}
@@ -152,6 +154,10 @@ void CkptMgrClient::SaveInstanceState(proto::ckptmgr::SaveInstanceStateRequest*
SendRequest(_request, NULL);
}
+void CkptMgrClient::SetPhysicalPlan(proto::system::PhysicalPlan& _pplan) {
+ pplan_ = &_pplan;
+}
+
void CkptMgrClient::GetInstanceState(const proto::system::Instance& _instance,
const std::string& _checkpoint_id) {
LOG(INFO) << "Sending GetInstanceState to ckptmgr for task_id " << _instance.info().task_id()
diff --git a/heron/stmgr/src/cpp/manager/ckptmgr-client.h b/heron/stmgr/src/cpp/manager/ckptmgr-client.h
index 2886cfb..4d34004 100644
--- a/heron/stmgr/src/cpp/manager/ckptmgr-client.h
+++ b/heron/stmgr/src/cpp/manager/ckptmgr-client.h
@@ -44,10 +44,10 @@ class CkptMgrClient : public Client {
void Quit();
- // TODO(nlu): add requests methods
virtual void SaveInstanceState(proto::ckptmgr::SaveInstanceStateRequest* _request);
virtual void GetInstanceState(const proto::system::Instance& _instance,
const std::string& _checkpoint_id);
+ virtual void SetPhysicalPlan(proto::system::PhysicalPlan& _pplan);
protected:
void GetInstanceState(const proto::system::Instance& _instance,
@@ -68,8 +68,6 @@ class CkptMgrClient : public Client {
void OnReconnectTimer();
- // TODO(nlu): add response handler methods
-
sp_string topology_name_;
sp_string topology_id_;
sp_string ckptmgr_id_;
@@ -83,6 +81,8 @@ class CkptMgrClient : public Client {
// Config
sp_int32 reconnect_cpktmgr_interval_sec_;
+
+ proto::system::PhysicalPlan* pplan_;
};
} // namespace stmgr
diff --git a/heron/stmgr/src/cpp/manager/stmgr.cpp b/heron/stmgr/src/cpp/manager/stmgr.cpp
index 4e4369e..973199c 100644
--- a/heron/stmgr/src/cpp/manager/stmgr.cpp
+++ b/heron/stmgr/src/cpp/manager/stmgr.cpp
@@ -130,12 +130,9 @@ void StMgr::Init() {
}
reliability_mode_ = heron::config::TopologyConfigHelper::GetReliabilityMode(*hydrated_topology_);
- if (reliability_mode_ == config::TopologyConfigVars::EFFECTIVELY_ONCE) {
- // Start checkpoint manager client
- CreateCheckpointMgrClient();
- } else {
- ckptmgr_client_ = nullptr;
- }
+ // create and start ckptmgr-client, stateful-restorer when receive pplan
+ ckptmgr_client_ = nullptr;
+ stateful_restorer_ = nullptr;
// Create the client manager
clientmgr_ = new StMgrClientMgr(eventLoop_, topology_name_, topology_id_, stmgr_id_, this,
@@ -166,18 +163,6 @@ void StMgr::Init() {
if (0 != metricscachemgr_mode_.compare("disabled")) {
FetchMetricsCacheLocation();
}
-
- if (reliability_mode_ == config::TopologyConfigVars::EFFECTIVELY_ONCE) {
- // Now start the stateful restorer
- stateful_restorer_ = new StatefulRestorer(ckptmgr_client_, clientmgr_,
- tuple_cache_, instance_server_, metrics_manager_client_,
- std::bind(&StMgr::HandleStatefulRestoreDone, this,
- std::placeholders::_1, std::placeholders::_2,
- std::placeholders::_3));
- } else {
- stateful_restorer_ = nullptr;
- }
-
// Check for log pruning every 5 minutes
CHECK_GT(eventLoop_->registerTimer(
[](EventLoop::Status) { ::heron::common::PruneLogs(); }, true,
@@ -353,7 +338,6 @@ void StMgr::CreateCheckpointMgrClient() {
topology_name_, topology_id_,
ckptmgr_id_, stmgr_id_,
save_watcher, get_watcher, ckpt_watcher);
- ckptmgr_client_->Start();
}
void StMgr::CreateTMasterClient(proto::tmaster::TMasterLocation* tmasterLocation) {
@@ -600,6 +584,20 @@ void StMgr::NewPhysicalPlan(proto::system::PhysicalPlan* _pplan) {
clientmgr_->StartConnections(pplan_);
}
instance_server_->BroadcastNewPhysicalPlan(*pplan_);
+
+ // create ckptmgr-client and stateful-restorer after receiving pplan
+ if (reliability_mode_ == config::TopologyConfigVars::EFFECTIVELY_ONCE
+ && ckptmgr_client_ == nullptr && stateful_restorer_ == nullptr) {
+ CreateCheckpointMgrClient();
+ stateful_restorer_ = new StatefulRestorer(ckptmgr_client_, clientmgr_,
+ tuple_cache_, instance_server_, metrics_manager_client_,
+ std::bind(&StMgr::HandleStatefulRestoreDone, this,
+ std::placeholders::_1, std::placeholders::_2,
+ std::placeholders::_3));
+
+ ckptmgr_client_->SetPhysicalPlan(*pplan_);
+ ckptmgr_client_->Start();
+ }
}
void StMgr::CleanupStreamConsumers() {
@@ -1003,7 +1001,7 @@ void StMgr::InitiateStatefulCheckpoint(sp_string _checkpoint_id) {
instance_server_->InitiateStatefulCheckpoint(_checkpoint_id);
}
-// We just recieved a InstanceStateCheckpoint message from one of our instances
+// We just received a InstanceStateCheckpoint message from one of our instances
// We need to propagate it to all downstream tasks
// We also need to send the checkpoint to ckptmgr
void StMgr::HandleStoreInstanceStateCheckpoint(