You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nl...@apache.org on 2018/07/27 21:31:56 UTC

[incubator-heron] branch master updated: spill state to local disk when configured (#2967)

This is an automated email from the ASF dual-hosted git repository.

nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git


The following commit(s) were added to refs/heads/master by this push:
     new 5c32f4d  spill state to local disk when configured (#2967)
5c32f4d is described below

commit 5c32f4dedd5b8ebd3daebb5d2f69732c070d4c7f
Author: Neng Lu <fr...@gmail.com>
AuthorDate: Fri Jul 27 14:31:53 2018 -0700

    spill state to local disk when configured (#2967)
    
    * spill state to local disk when configured
    
    * start ckptmgr-client after receiving pplan
    
    * separate instances and ckptmgr spilled location
    
    * clean states by owner
    
    * bail out if spill state failed
    
    * typo
---
 .../api/StatefulTumblingWindowTopology.java        |  4 +-
 .../api/src/java/org/apache/heron/api/Config.java  | 30 ++++++++
 .../java/org/apache/heron/api/HeronTopology.java   |  6 ++
 heron/ckptmgr/src/java/BUILD                       |  8 ++-
 .../heron/ckptmgr/CheckpointManagerServer.java     | 79 ++++++++++++++++++++--
 .../heron/ckptmgr/CheckpointManagerServerTest.java | 14 ++++
 .../heron/instance/AbstractOutputCollector.java    |  6 +-
 .../heron/instance/OutgoingTupleCollection.java    | 36 +++++++---
 .../src/java/org/apache/heron/instance/Slave.java  | 10 +++
 .../apache/heron/instance/bolt/BoltInstance.java   | 31 +++++++--
 .../apache/heron/instance/spout/SpoutInstance.java | 22 +++++-
 heron/proto/ckptmgr.proto                          |  7 +-
 heron/stmgr/src/cpp/manager/ckptmgr-client.cpp     |  6 ++
 heron/stmgr/src/cpp/manager/ckptmgr-client.h       |  6 +-
 heron/stmgr/src/cpp/manager/stmgr.cpp              | 38 +++++------
 15 files changed, 250 insertions(+), 53 deletions(-)

diff --git a/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java b/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
index b39e3ff..62c8185 100644
--- a/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
+++ b/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
@@ -155,7 +155,8 @@ public final class StatefulTumblingWindowTopology {
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("integer", new IntegerSpout(), 1);
     WindowSumBolt windowSumBolt = new WindowSumBolt();
-    windowSumBolt.withTumblingWindow(Duration.ofSeconds(10));
+    windowSumBolt.withTumblingWindow(Duration.ofMinutes(10));
+
     builder.setBolt("sumbolt", windowSumBolt, 1).shuffleGrouping("integer");
     builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("sumbolt");
     Config conf = new Config();
@@ -172,6 +173,7 @@ public final class StatefulTumblingWindowTopology {
     conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
     conf.setTopologyStatefulCheckpointIntervalSecs(20);
     conf.setMaxSpoutPending(1000);
+    conf.setMessageTimeoutSecs(1500);
 
     if (args != null && args.length > 0) {
       topoName = args[0];
diff --git a/heron/api/src/java/org/apache/heron/api/Config.java b/heron/api/src/java/org/apache/heron/api/Config.java
index 2e22731..d91c8a0 100644
--- a/heron/api/src/java/org/apache/heron/api/Config.java
+++ b/heron/api/src/java/org/apache/heron/api/Config.java
@@ -223,6 +223,16 @@ public class Config extends HashMap<String, Object> {
   public static final String TOPOLOGY_STATEFUL_CKPTMGR_RAM =
                             "topology.stateful.checkpointmanager.ram";
   /**
+   * Whether spill the state to disk for transferring
+   */
+  public static final String TOPOLOGY_STATEFUL_SPILL_STATE =
+                            "topology.stateful.spill.state";
+  /**
+   * The local disk location where the state is spilled
+   */
+  public static final String TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION =
+                            "topology.stateful.spill.state.location";
+  /**
    * Stream Manager RAM requirement
    */
   public static final String TOPOLOGY_STMGR_RAM =  "topology.stmgr.ram";
@@ -333,6 +343,8 @@ public class Config extends HashMap<String, Object> {
     apiVars.add(TOPOLOGY_STATEFUL_START_CLEAN);
     apiVars.add(TOPOLOGY_STATEFUL_CHECKPOINT_INTERVAL_SECONDS);
     apiVars.add(TOPOLOGY_STATEFUL_CKPTMGR_RAM);
+    apiVars.add(TOPOLOGY_STATEFUL_SPILL_STATE);
+    apiVars.add(TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION);
     apiVars.add(TOPOLOGY_STMGR_RAM);
     apiVars.add(TOPOLOGY_METRICSMGR_RAM);
     apiVars.add(TOPOLOGY_RELIABILITY_MODE);
@@ -593,6 +605,16 @@ public class Config extends HashMap<String, Object> {
     conf.put(Config.TOPOLOGY_STATEFUL_CKPTMGR_RAM, ramInBytes.asBytes());
   }
 
+  public static void setTopologyStatefulSpillState(Map<String, Object> conf,
+                                   String spillState) {
+    conf.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE, spillState);
+  }
+
+  public static void setTopologyStatefulSpillStateLocation(Map<String, Object> conf,
+                                                           String location) {
+    conf.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION, location);
+  }
+
   public static void setStreamManagerRam(Map<String, Object> conf,
                                          ByteAmount ramInBytes) {
     conf.put(Config.TOPOLOGY_STMGR_RAM, ramInBytes.asBytes());
@@ -773,6 +795,14 @@ public class Config extends HashMap<String, Object> {
     setCheckpointManagerRam(this, ramInBytes);
   }
 
+  public void setTopologyStatefulSpillState(String spillState) {
+    setTopologyStatefulSpillState(this, spillState);
+  }
+
+  public void setTopologyStatefulSpillStateLocation(String location) {
+    setTopologyStatefulSpillStateLocation(this, location);
+  }
+
   public void setStreamManagerRam(ByteAmount ramInBytes) {
     setStreamManagerRam(this, ramInBytes);
   }
diff --git a/heron/api/src/java/org/apache/heron/api/HeronTopology.java b/heron/api/src/java/org/apache/heron/api/HeronTopology.java
index e0aaeff..a6263f7 100644
--- a/heron/api/src/java/org/apache/heron/api/HeronTopology.java
+++ b/heron/api/src/java/org/apache/heron/api/HeronTopology.java
@@ -62,6 +62,12 @@ public class HeronTopology {
     if (!userConfig.containsKey(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS)) {
       userConfig.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, "true");
     }
+    if (!userConfig.containsKey(Config.TOPOLOGY_STATEFUL_SPILL_STATE)) {
+      userConfig.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE, "false");
+      if (!userConfig.containsKey(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION)) {
+        userConfig.put(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION, "./spilled-state/");
+      }
+    }
   }
 
   public TopologyAPI.Topology getTopology() {
diff --git a/heron/ckptmgr/src/java/BUILD b/heron/ckptmgr/src/java/BUILD
index 5a7799b..e139b13 100644
--- a/heron/ckptmgr/src/java/BUILD
+++ b/heron/ckptmgr/src/java/BUILD
@@ -9,13 +9,17 @@ java_library(
         exclude = ["**/CheckpointManager.java"],
     ),
     deps = [
-        "//heron/spi/src/java:statefulstorage-spi-java",
+        "//heron/api/src/java:api-java-low-level",
         "//heron/common/src/java:basics-java",
         "//heron/common/src/java:config-java",
         "//heron/common/src/java:network-java",
         "//heron/common/src/java:utils-java",
         "//heron/proto:proto_common_java",
         "//heron/proto:proto_ckptmgr_java",
+        "//heron/proto:proto_physical_plan_java",
+        "//heron/proto:proto_stmgr_java",
+        "//heron/proto:proto_topology_java",
+        "//heron/spi/src/java:statefulstorage-spi-java",
         "@com_google_protobuf//:protobuf_java",
     ],
 )
@@ -25,13 +29,13 @@ java_binary(
     srcs = glob(["**/CheckpointManager.java"]),
     deps = [
         ":ckptmgr-java",
-        "//heron/spi/src/java:statefulstorage-spi-java",
         "//heron/api/src/java:api-java-low-level",
         "//heron/common/src/java:basics-java",
         "//heron/common/src/java:config-java",
         "//heron/common/src/java:network-java",
         "//heron/common/src/java:utils-java",
         "//heron/proto:proto_ckptmgr_java",
+        "//heron/spi/src/java:statefulstorage-spi-java",
         "@commons_cli_commons_cli//jar",
     ],
 )
diff --git a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
index 4b4654a..f6c3416 100644
--- a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
+++ b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
@@ -21,18 +21,24 @@ package org.apache.heron.ckptmgr;
 
 import java.io.IOException;
 import java.nio.channels.SocketChannel;
+import java.util.UUID;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import com.google.protobuf.ByteString;
 import com.google.protobuf.Message;
 
+import org.apache.heron.api.Config;
+import org.apache.heron.api.generated.TopologyAPI;
+import org.apache.heron.api.utils.TopologyUtils;
+import org.apache.heron.common.basics.FileUtils;
 import org.apache.heron.common.basics.NIOLooper;
 import org.apache.heron.common.network.HeronServer;
 import org.apache.heron.common.network.HeronSocketOptions;
 import org.apache.heron.common.network.REQID;
 import org.apache.heron.proto.ckptmgr.CheckpointManager;
 import org.apache.heron.proto.system.Common;
+import org.apache.heron.proto.system.PhysicalPlans;
 import org.apache.heron.spi.statefulstorage.Checkpoint;
 import org.apache.heron.spi.statefulstorage.CheckpointInfo;
 import org.apache.heron.spi.statefulstorage.IStatefulStorage;
@@ -45,6 +51,8 @@ public class CheckpointManagerServer extends HeronServer {
   private final String topologyId;
   private final String checkpointMgrId;
   private final IStatefulStorage statefulStorage;
+  private boolean spillState;
+  private String spillStateLocation;
 
   private SocketChannel connection;
 
@@ -63,6 +71,8 @@ public class CheckpointManagerServer extends HeronServer {
     this.topologyId = topologyId;
     this.checkpointMgrId = checkpointMgrId;
     this.statefulStorage = statefulStorage;
+    this.spillState = false;
+    this.spillStateLocation = "";
 
     this.connection = null;
 
@@ -108,6 +118,10 @@ public class CheckpointManagerServer extends HeronServer {
     }
   }
 
+  @Override
+  public void onMessage(SocketChannel channel, Message message) {
+  }
+
   protected void handleCleanStatefulCheckpointRequest(
       REQID rid,
       SocketChannel channel,
@@ -178,6 +192,7 @@ public class CheckpointManagerServer extends HeronServer {
   ) {
     LOG.info(String.format("Got a StMgr register request from %s running on host:port %s",
                            request.getStmgrId(), channel.socket().getRemoteSocketAddress()));
+    handlePhysicalPlan(request.getPhysicalPlan());
 
     CheckpointManager.RegisterStMgrResponse.Builder responseBuilder =
         CheckpointManager.RegisterStMgrResponse.newBuilder();
@@ -210,8 +225,15 @@ public class CheckpointManagerServer extends HeronServer {
   ) {
     CheckpointInfo info =
         new CheckpointInfo(request.getCheckpoint().getCheckpointId(),
-                           request.getInstance());
-    Checkpoint checkpoint = new Checkpoint(request.getCheckpoint());
+            request.getInstance());
+
+    Checkpoint checkpoint;
+    if (request.getCheckpoint().hasStateLocation()) {
+      checkpoint = loadCheckpoint(request.getInstance(), request.getCheckpoint().getCheckpointId(),
+          request.getCheckpoint().getStateLocation());
+    } else {
+      checkpoint = new Checkpoint(request.getCheckpoint());
+    }
 
     LOG.info(String.format("Got a save checkpoint request for checkpointId %s "
                            + " component %s instanceId %s on connection %s",
@@ -246,6 +268,18 @@ public class CheckpointManagerServer extends HeronServer {
     sendResponse(rid, channel, responseBuilder.build());
   }
 
+  private Checkpoint loadCheckpoint(PhysicalPlans.Instance instanceInfo,
+                                    String checkpointId, String localStateLocation) {
+    LOG.info("fetch state from: " + localStateLocation);
+    CheckpointManager.InstanceStateCheckpoint checkpoint =
+        CheckpointManager.InstanceStateCheckpoint.newBuilder()
+            .setCheckpointId(checkpointId)
+            .setState(ByteString.copyFrom(FileUtils.readFromFile(localStateLocation)))
+            .build();
+
+    return new Checkpoint(checkpoint);
+  }
+
   protected void handleGetInstanceStateRequest(
       REQID rid,
       SocketChannel channel,
@@ -283,7 +317,28 @@ public class CheckpointManagerServer extends HeronServer {
                                info.getComponent(),
                                info.getInstanceId()));
         // Set the checkpoint-state in response
-        responseBuilder.setCheckpoint(checkpoint.getCheckpoint());
+        if (spillState) {
+          CheckpointManager.InstanceStateCheckpoint ckpt = checkpoint.getCheckpoint();
+          String checkpointId = ckpt.getCheckpointId();
+
+          // clean any possible existing states
+          FileUtils.cleanDir(spillStateLocation);
+
+          // spill state to local disk
+          String stateLocation = spillStateLocation + checkpointId + "-" + UUID.randomUUID();
+          if (!FileUtils.writeToFile(stateLocation, ckpt.getState().toByteArray(), true)) {
+            throw new RuntimeException("failed to spill state. Bailing out...");
+          }
+          LOG.info("spilled state to: " + stateLocation);
+          ckpt = CheckpointManager.InstanceStateCheckpoint.newBuilder()
+                  .setStateLocation(stateLocation)
+                  .setCheckpointId(checkpointId)
+                  .build();
+
+          responseBuilder.setCheckpoint(ckpt);
+        } else {
+          responseBuilder.setCheckpoint(checkpoint.getCheckpoint());
+        }
       } catch (StatefulStorageException e) {
         errorMessage = String.format("Get checkpoint not successful for checkpointId %s "
                                      + "component %s instanceId %d",
@@ -301,9 +356,21 @@ public class CheckpointManagerServer extends HeronServer {
     sendResponse(rid, channel, responseBuilder.build());
   }
 
-  @Override
-  public void onMessage(SocketChannel channel, Message message) {
-
+  private void handlePhysicalPlan(PhysicalPlans.PhysicalPlan pplan) {
+    TopologyAPI.Config config = pplan.getTopology().getTopologyConfig();
+    this.spillState = Boolean.parseBoolean(TopologyUtils.getConfigWithDefault(config.getKvsList(),
+        Config.TOPOLOGY_STATEFUL_SPILL_STATE, "false"));
+    this.spillStateLocation = String.format("%s/%s/",
+        String.valueOf(TopologyUtils.getConfigWithDefault(config.getKvsList(),
+                       Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION, "")),
+        "ckptmgr");
+    if (FileUtils.isDirectoryExists(spillStateLocation)) {
+      FileUtils.cleanDir(spillStateLocation);
+    } else {
+      FileUtils.createDirectory(spillStateLocation);
+    }
+    LOG.info("spill state: " + spillState
+        + ". set spilled state location: " + spillStateLocation);
   }
 
   @Override
diff --git a/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java b/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java
index c6bffc7..34eba63 100644
--- a/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java
+++ b/heron/ckptmgr/tests/java/org/apache/heron/ckptmgr/CheckpointManagerServerTest.java
@@ -30,6 +30,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import org.apache.heron.api.generated.TopologyAPI;
 import org.apache.heron.common.basics.NIOLooper;
 import org.apache.heron.common.basics.SysUtils;
 import org.apache.heron.common.network.HeronClient;
@@ -79,6 +80,8 @@ public class CheckpointManagerServerTest {
     final int COMPONENT_INDEX = 1;
     final String COMPONENT_NAME = "component_name";
     final byte[] BYTES = "checkpoint manager server test bytes".getBytes();
+    final String TOPO_ID = "topo_id";
+    final String TOPO_NAME = "topo_name";
 
     PhysicalPlans.InstanceInfo info = PhysicalPlans.InstanceInfo.newBuilder()
         .setTaskId(TASK_ID)
@@ -86,6 +89,16 @@ public class CheckpointManagerServerTest {
         .setComponentName(COMPONENT_NAME)
         .build();
 
+    TopologyAPI.Topology topology = TopologyAPI.Topology.newBuilder()
+        .setId(TOPO_ID)
+        .setName(TOPO_NAME)
+        .setState(TopologyAPI.TopologyState.RUNNING)
+        .build();
+
+    PhysicalPlans.PhysicalPlan pplan = PhysicalPlans.PhysicalPlan.newBuilder()
+        .setTopology(topology)
+        .build();
+
     instance = PhysicalPlans.Instance.newBuilder()
         .setInstanceId(INSTANCE_ID)
         .setStmgrId(STMGR_ID)
@@ -121,6 +134,7 @@ public class CheckpointManagerServerTest {
         .setTopologyId(TOPOLOGY_ID)
         .setStmgrId(STMGR_ID)
         .setTopologyName(TOPOLOGY_NAME)
+        .setPhysicalPlan(pplan)
         .build();
 
     registerTMasterRequest = CheckpointManager.RegisterTMasterRequest.newBuilder()
diff --git a/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java b/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java
index 8d5f1fa..08afa61 100644
--- a/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java
+++ b/heron/instance/src/java/org/apache/heron/instance/AbstractOutputCollector.java
@@ -114,8 +114,10 @@ public class AbstractOutputCollector {
 
   // Flush the states
   public void sendOutState(State<Serializable, Serializable> state,
-                           String checkpointId) {
-    outputter.sendOutState(state, checkpointId);
+                           String checkpointId,
+                           boolean spillState,
+                           String location) {
+    outputter.sendOutState(state, checkpointId, spillState, location);
   }
 
   // Clean the internal state of BoltOutputCollectorImpl
diff --git a/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java b/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java
index 7c28290..862c300 100644
--- a/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java
+++ b/heron/instance/src/java/org/apache/heron/instance/OutgoingTupleCollection.java
@@ -20,6 +20,7 @@
 package org.apache.heron.instance;
 
 import java.io.Serializable;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -31,6 +32,7 @@ import org.apache.heron.api.serializer.IPluggableSerializer;
 import org.apache.heron.api.state.State;
 import org.apache.heron.common.basics.ByteAmount;
 import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
 import org.apache.heron.common.basics.SingletonRegistry;
 import org.apache.heron.common.config.SystemConfig;
 import org.apache.heron.common.utils.misc.PhysicalPlanHelper;
@@ -102,30 +104,44 @@ public class OutgoingTupleCollection {
   }
 
   /**
-   * Send out the instance's state with corresponding checkpointId
+   * Send out the instance's state with corresponding checkpointId. If spillState is True,
+   * the actual state is spill to disk and only the state location is sent out.
    * @param state instance's state
    * @param checkpointId the checkpointId
+   * @param spillState spill the state to local disk if true
+   * @param location the location where state is spilled
    */
   public void sendOutState(State<Serializable, Serializable> state,
-                           String checkpointId) {
+                           String checkpointId,
+                           boolean spillState,
+                           String location) {
     lock.lock();
     try {
       // flush all the current data before sending the state
       flushRemaining();
 
-      // Serialize the state
+      // serialize the state
       byte[] serializedState = serializer.serialize(state);
 
-      // Construct the instance state checkpoint
-      CheckpointManager.InstanceStateCheckpoint instanceState =
-          CheckpointManager.InstanceStateCheckpoint.newBuilder()
-              .setCheckpointId(checkpointId)
-              .setState(ByteString.copyFrom(serializedState))
-              .build();
+      CheckpointManager.InstanceStateCheckpoint.Builder instanceStateBuilder =
+          CheckpointManager.InstanceStateCheckpoint.newBuilder();
+      instanceStateBuilder.setCheckpointId(checkpointId);
+
+      if (spillState) {
+        FileUtils.cleanDir(location);
+
+        String stateLocation = location + checkpointId + "-" + UUID.randomUUID();
+        if (!FileUtils.writeToFile(stateLocation, serializedState, true)) {
+          throw new RuntimeException("failed to spill state. Bailing out...");
+        }
+        instanceStateBuilder.setStateLocation(stateLocation);
+      } else {
+        instanceStateBuilder.setState(ByteString.copyFrom(serializedState));
+      }
 
       CheckpointManager.StoreInstanceStateCheckpoint storeRequest =
           CheckpointManager.StoreInstanceStateCheckpoint.newBuilder()
-              .setState(instanceState)
+              .setState(instanceStateBuilder.build())
               .build();
 
       // Put the checkpoint to out stream queue
diff --git a/heron/instance/src/java/org/apache/heron/instance/Slave.java b/heron/instance/src/java/org/apache/heron/instance/Slave.java
index 7d8e1f8..5b85fc6 100644
--- a/heron/instance/src/java/org/apache/heron/instance/Slave.java
+++ b/heron/instance/src/java/org/apache/heron/instance/Slave.java
@@ -32,6 +32,7 @@ import org.apache.heron.api.serializer.IPluggableSerializer;
 import org.apache.heron.api.state.HashMapState;
 import org.apache.heron.api.state.State;
 import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
 import org.apache.heron.common.basics.SingletonRegistry;
 import org.apache.heron.common.basics.SlaveLooper;
 import org.apache.heron.common.config.SystemConfig;
@@ -304,6 +305,7 @@ public class Slave implements Runnable, AutoCloseable {
       instanceState.clear();
       instanceState = null;
     }
+
     if (request.getState().hasState() && !request.getState().getState().isEmpty()) {
       @SuppressWarnings("unchecked")
       State<Serializable, Serializable> stateToRestore =
@@ -311,6 +313,14 @@ public class Slave implements Runnable, AutoCloseable {
               request.getState().getState().toByteArray());
 
       instanceState = stateToRestore;
+    } else if (request.getState().hasStateLocation()) {
+      String stateLocation = request.getState().getStateLocation();
+      byte[] rawState = FileUtils.readFromFile(stateLocation);
+
+      @SuppressWarnings("unchecked")
+      State<Serializable, Serializable> stateToRestore =
+          (State<Serializable, Serializable>) serializer.deserialize(rawState);
+      instanceState = stateToRestore;
     } else {
       LOG.info("The restore request does not have an actual state");
     }
diff --git a/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java b/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
index cd49dac..f58e181 100644
--- a/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
+++ b/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
@@ -40,6 +40,7 @@ import org.apache.heron.api.topology.IStatefulComponent;
 import org.apache.heron.api.topology.IUpdatable;
 import org.apache.heron.api.utils.Utils;
 import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
 import org.apache.heron.common.basics.SingletonRegistry;
 import org.apache.heron.common.basics.SlaveLooper;
 import org.apache.heron.common.basics.TypeUtils;
@@ -68,13 +69,21 @@ public class BoltInstance implements IInstance {
   private final Communicator<Message> streamInQueue;
 
   private final boolean isTopologyStateful;
+  private final boolean spillState;
+  private final String spillStateLocation;
 
   private State<Serializable, Serializable> instanceState;
 
+  // The reference to topology's config
+  private final Map<String, Object> config;
+
   private final SlaveLooper looper;
 
   private final SystemConfig systemConfig;
 
+  /**
+   * Construct a BoltInstance basing on given arguments
+   */
   public BoltInstance(PhysicalPlanHelper helper,
                       Communicator<Message> streamInQueue,
                       Communicator<Message> streamOutQueue,
@@ -89,12 +98,18 @@ public class BoltInstance implements IInstance {
     this.systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(
         SystemConfig.HERON_SYSTEM_CONFIG);
 
-    Map<String, Object> config = helper.getTopologyContext().getTopologyConfig();
+    this.config = helper.getTopologyContext().getTopologyConfig();
     this.isTopologyStateful = String.valueOf(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE)
         .equals(config.get(Config.TOPOLOGY_RELIABILITY_MODE));
-
     LOG.info("Is this topology stateful: " + isTopologyStateful);
 
+    this.spillState =
+        Boolean.parseBoolean((String) config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE));
+
+    this.spillStateLocation = String.format("%s/%s/",
+            String.valueOf(config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION)),
+            helper.getMyInstanceId());
+
     if (helper.getMyBolt() == null) {
       throw new RuntimeException("HeronBoltInstance has no bolt in physical plan.");
     }
@@ -143,7 +158,6 @@ public class BoltInstance implements IInstance {
     if (!isTopologyStateful) {
       throw new RuntimeException("Could not save a non-stateful topology's state");
     }
-
     // need to synchronize with other OutgoingTupleCollection operations
     // so that topology emit, ack, fail are thread safe
     collector.lock.lock();
@@ -152,8 +166,7 @@ public class BoltInstance implements IInstance {
       if (bolt instanceof IStatefulComponent) {
         ((IStatefulComponent) bolt).preSave(checkpointId);
       }
-
-      collector.sendOutState(instanceState, checkpointId);
+      collector.sendOutState(instanceState, checkpointId, spillState, spillStateLocation);
     } finally {
       collector.lock.unlock();
     }
@@ -174,6 +187,14 @@ public class BoltInstance implements IInstance {
     if (bolt instanceof IStatefulComponent) {
       this.instanceState = state;
       ((IStatefulComponent<Serializable, Serializable>) bolt).initState(instanceState);
+
+      if (spillState) {
+        if (FileUtils.isDirectoryExists(spillStateLocation)) {
+          FileUtils.cleanDir(spillStateLocation);
+        } else {
+          FileUtils.createDirectory(spillStateLocation);
+        }
+      }
     }
 
     // Delegate
diff --git a/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java b/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
index f7c077a..5fc1fdb 100644
--- a/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
+++ b/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
@@ -40,6 +40,7 @@ import org.apache.heron.api.topology.IUpdatable;
 import org.apache.heron.api.utils.Utils;
 import org.apache.heron.common.basics.ByteAmount;
 import org.apache.heron.common.basics.Communicator;
+import org.apache.heron.common.basics.FileUtils;
 import org.apache.heron.common.basics.SingletonRegistry;
 import org.apache.heron.common.basics.SlaveLooper;
 import org.apache.heron.common.basics.TypeUtils;
@@ -67,6 +68,9 @@ public class SpoutInstance implements IInstance {
   private final boolean enableMessageTimeouts;
 
   private final boolean isTopologyStateful;
+  private final boolean spillState;
+  private final String spillStateLocation;
+
   private State<Serializable, Serializable> instanceState;
 
   private final SlaveLooper looper;
@@ -99,6 +103,13 @@ public class SpoutInstance implements IInstance {
     this.isTopologyStateful = String.valueOf(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE)
         .equals(config.get(Config.TOPOLOGY_RELIABILITY_MODE));
 
+    this.spillState =
+        Boolean.parseBoolean((String) config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE));
+
+    this.spillStateLocation = String.format("%s/%s/",
+        String.valueOf(config.get(Config.TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION)),
+        helper.getMyInstanceId());
+
     LOG.info("Is this topology stateful: " + isTopologyStateful);
 
     if (helper.getMySpout() == null) {
@@ -160,8 +171,7 @@ public class SpoutInstance implements IInstance {
       if (spout instanceof IStatefulComponent) {
         ((IStatefulComponent) spout).preSave(checkpointId);
       }
-
-      collector.sendOutState(instanceState, checkpointId);
+      collector.sendOutState(instanceState, checkpointId, spillState, spillStateLocation);
     } finally {
       collector.lock.unlock();
     }
@@ -182,6 +192,14 @@ public class SpoutInstance implements IInstance {
     if (spout instanceof IStatefulComponent) {
       this.instanceState = state;
       ((IStatefulComponent<Serializable, Serializable>) spout).initState(instanceState);
+
+      if (spillState) {
+        if (FileUtils.isDirectoryExists(spillStateLocation)) {
+          FileUtils.cleanDir(spillStateLocation);
+        } else {
+          FileUtils.createDirectory(spillStateLocation);
+        }
+      }
     }
 
     spout.open(
diff --git a/heron/proto/ckptmgr.proto b/heron/proto/ckptmgr.proto
index 68935cc..f21e6ed 100644
--- a/heron/proto/ckptmgr.proto
+++ b/heron/proto/ckptmgr.proto
@@ -180,10 +180,12 @@ message CleanStatefulCheckpointResponse {
  */
 
 // This message encapsulates the info associated with
-// state of an instance/partition
+// state of an instance/partition. only one of the state_location
+// and state should be set
 message InstanceStateCheckpoint {
   required string checkpoint_id = 1;
-  required bytes state = 2;
+  optional bytes state = 2;
+  optional string state_location = 3;
 }
 
 // This message encapsulates the info associated with
@@ -200,6 +202,7 @@ message RegisterStMgrRequest {
   required string topology_name = 1;
   required string topology_id = 2;
   required string stmgr_id = 3;
+  required heron.proto.system.PhysicalPlan physical_plan = 4;
 }
 
 // This is the message that checkpoint manager
diff --git a/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp b/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp
index c6e6a04..a41bf9b 100644
--- a/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp
+++ b/heron/stmgr/src/cpp/manager/ckptmgr-client.cpp
@@ -42,6 +42,7 @@ CkptMgrClient::CkptMgrClient(EventLoop* eventloop, const NetworkOptions& _option
       ckptmgr_id_(_ckptmgr_id),
       stmgr_id_(_stmgr_id),
       quit_(false),
+      pplan_(nullptr),
       ckpt_saved_watcher_(_ckpt_saved_watcher),
       ckpt_get_watcher_(_ckpt_get_watcher),
       register_watcher_(_register_watcher) {
@@ -144,6 +145,7 @@ void CkptMgrClient::SendRegisterRequest() {
   request->set_topology_name(topology_name_);
   request->set_topology_id(topology_id_);
   request->set_stmgr_id(stmgr_id_);
+  request->mutable_physical_plan()->CopyFrom(*pplan_);
   SendRequest(request, NULL);
 }
 
@@ -152,6 +154,10 @@ void CkptMgrClient::SaveInstanceState(proto::ckptmgr::SaveInstanceStateRequest*
   SendRequest(_request, NULL);
 }
 
+void CkptMgrClient::SetPhysicalPlan(proto::system::PhysicalPlan& _pplan) {
+  pplan_ = &_pplan;
+}
+
 void CkptMgrClient::GetInstanceState(const proto::system::Instance& _instance,
                                      const std::string& _checkpoint_id) {
   LOG(INFO) << "Sending GetInstanceState to ckptmgr for task_id " << _instance.info().task_id()
diff --git a/heron/stmgr/src/cpp/manager/ckptmgr-client.h b/heron/stmgr/src/cpp/manager/ckptmgr-client.h
index 2886cfb..4d34004 100644
--- a/heron/stmgr/src/cpp/manager/ckptmgr-client.h
+++ b/heron/stmgr/src/cpp/manager/ckptmgr-client.h
@@ -44,10 +44,10 @@ class CkptMgrClient : public Client {
 
   void Quit();
 
-  // TODO(nlu): add requests methods
   virtual void SaveInstanceState(proto::ckptmgr::SaveInstanceStateRequest* _request);
   virtual void GetInstanceState(const proto::system::Instance& _instance,
                                 const std::string& _checkpoint_id);
+  virtual void SetPhysicalPlan(proto::system::PhysicalPlan& _pplan);
 
  protected:
   void GetInstanceState(const proto::system::Instance& _instance,
@@ -68,8 +68,6 @@ class CkptMgrClient : public Client {
 
   void OnReconnectTimer();
 
-  // TODO(nlu): add response handler methods
-
   sp_string topology_name_;
   sp_string topology_id_;
   sp_string ckptmgr_id_;
@@ -83,6 +81,8 @@ class CkptMgrClient : public Client {
 
   // Config
   sp_int32 reconnect_cpktmgr_interval_sec_;
+
+  proto::system::PhysicalPlan* pplan_;
 };
 
 }  // namespace stmgr
diff --git a/heron/stmgr/src/cpp/manager/stmgr.cpp b/heron/stmgr/src/cpp/manager/stmgr.cpp
index 4e4369e..973199c 100644
--- a/heron/stmgr/src/cpp/manager/stmgr.cpp
+++ b/heron/stmgr/src/cpp/manager/stmgr.cpp
@@ -130,12 +130,9 @@ void StMgr::Init() {
   }
 
   reliability_mode_ = heron::config::TopologyConfigHelper::GetReliabilityMode(*hydrated_topology_);
-  if (reliability_mode_ == config::TopologyConfigVars::EFFECTIVELY_ONCE) {
-    // Start checkpoint manager client
-    CreateCheckpointMgrClient();
-  } else {
-    ckptmgr_client_ = nullptr;
-  }
+  // create and start ckptmgr-client, stateful-restorer when receive pplan
+  ckptmgr_client_ = nullptr;
+  stateful_restorer_ = nullptr;
 
   // Create the client manager
   clientmgr_ = new StMgrClientMgr(eventLoop_, topology_name_, topology_id_, stmgr_id_, this,
@@ -166,18 +163,6 @@ void StMgr::Init() {
   if (0 != metricscachemgr_mode_.compare("disabled")) {
     FetchMetricsCacheLocation();
   }
-
-  if (reliability_mode_ == config::TopologyConfigVars::EFFECTIVELY_ONCE) {
-    // Now start the stateful restorer
-    stateful_restorer_ = new StatefulRestorer(ckptmgr_client_, clientmgr_,
-                               tuple_cache_, instance_server_, metrics_manager_client_,
-                               std::bind(&StMgr::HandleStatefulRestoreDone, this,
-                                         std::placeholders::_1, std::placeholders::_2,
-                                         std::placeholders::_3));
-  } else {
-    stateful_restorer_ = nullptr;
-  }
-
   // Check for log pruning every 5 minutes
   CHECK_GT(eventLoop_->registerTimer(
                [](EventLoop::Status) { ::heron::common::PruneLogs(); }, true,
@@ -353,7 +338,6 @@ void StMgr::CreateCheckpointMgrClient() {
                                       topology_name_, topology_id_,
                                       ckptmgr_id_, stmgr_id_,
                                       save_watcher, get_watcher, ckpt_watcher);
-  ckptmgr_client_->Start();
 }
 
 void StMgr::CreateTMasterClient(proto::tmaster::TMasterLocation* tmasterLocation) {
@@ -600,6 +584,20 @@ void StMgr::NewPhysicalPlan(proto::system::PhysicalPlan* _pplan) {
     clientmgr_->StartConnections(pplan_);
   }
   instance_server_->BroadcastNewPhysicalPlan(*pplan_);
+
+  // create ckptmgr-client and stateful-restorer after receiving pplan
+  if (reliability_mode_ == config::TopologyConfigVars::EFFECTIVELY_ONCE
+      && ckptmgr_client_ == nullptr && stateful_restorer_ == nullptr) {
+    CreateCheckpointMgrClient();
+    stateful_restorer_ = new StatefulRestorer(ckptmgr_client_, clientmgr_,
+                               tuple_cache_, instance_server_, metrics_manager_client_,
+                               std::bind(&StMgr::HandleStatefulRestoreDone, this,
+                                         std::placeholders::_1, std::placeholders::_2,
+                                         std::placeholders::_3));
+
+    ckptmgr_client_->SetPhysicalPlan(*pplan_);
+    ckptmgr_client_->Start();
+  }
 }
 
 void StMgr::CleanupStreamConsumers() {
@@ -1003,7 +1001,7 @@ void StMgr::InitiateStatefulCheckpoint(sp_string _checkpoint_id) {
   instance_server_->InitiateStatefulCheckpoint(_checkpoint_id);
 }
 
-// We just recieved a InstanceStateCheckpoint message from one of our instances
+// We just received a InstanceStateCheckpoint message from one of our instances
 // We need to propagate it to all downstream tasks
 // We also need to send the checkpoint to ckptmgr
 void StMgr::HandleStoreInstanceStateCheckpoint(